These NEON and non-NEON implementations come from Andy Polyakov's
implementation, with several modifications for being friendly to kernel
space.

Signed-off-by: Jason A. Donenfeld <ja...@zx2c4.com>
Cc: Andy Lutomirski <l...@kernel.org>
Cc: Greg KH <gre...@linuxfoundation.org>
Cc: Samuel Neves <sne...@dei.uc.pt>
Cc: Jean-Philippe Aumasson <jeanphilippe.aumas...@gmail.com>
Cc: Andy Polyakov <ap...@openssl.org>
Cc: Russell King <li...@armlinux.org.uk>
Cc: linux-arm-ker...@lists.infradead.org
Cc: linux-crypto@vger.kernel.org
---
 lib/zinc/Makefile                     |    8 +
 lib/zinc/chacha20/chacha20-arm-glue.h |   42 +
 lib/zinc/chacha20/chacha20-arm.S      | 1471 +++++++++++++++++++
 lib/zinc/chacha20/chacha20-arm64.S    | 1940 +++++++++++++++++++++++++
 4 files changed, 3461 insertions(+)
 create mode 100644 lib/zinc/chacha20/chacha20-arm-glue.h
 create mode 100644 lib/zinc/chacha20/chacha20-arm.S
 create mode 100644 lib/zinc/chacha20/chacha20-arm64.S

diff --git a/lib/zinc/Makefile b/lib/zinc/Makefile
index 6ec77feb4ab7..87be627f3354 100644
--- a/lib/zinc/Makefile
+++ b/lib/zinc/Makefile
@@ -4,6 +4,14 @@ ccflags-y += -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt'
 
 ifeq ($(CONFIG_ZINC_CHACHA20),y)
 zinc-y += chacha20/chacha20.o
+ifeq ($(CONFIG_ARM),y)
+zinc-y += chacha20/chacha20-arm.o
+CFLAGS_chacha20.o += -include $(srctree)/$(src)/chacha20/chacha20-arm-glue.h
+endif
+ifeq ($(CONFIG_ARM64),y)
+zinc-y += chacha20/chacha20-arm64.o
+CFLAGS_chacha20.o += -include $(srctree)/$(src)/chacha20/chacha20-arm-glue.h
+endif
 endif
 
 zinc-y += main.o
diff --git a/lib/zinc/chacha20/chacha20-arm-glue.h 
b/lib/zinc/chacha20/chacha20-arm-glue.h
new file mode 100644
index 000000000000..1b072eb1e50c
--- /dev/null
+++ b/lib/zinc/chacha20/chacha20-arm-glue.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2015-2018 Jason A. Donenfeld <ja...@zx2c4.com>. All Rights 
Reserved.
+ */
+
+#include <zinc/chacha20.h>
+
+asmlinkage void chacha20_arm(u8 *out, const u8 *in, const size_t len, const 
u32 key[8], const u32 counter[4]);
+#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (!defined(__LINUX_ARM_ARCH__) || 
__LINUX_ARM_ARCH__ >= 7)
+#define ARM_USE_NEON
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len, const 
u32 key[8], const u32 counter[4]);
+#endif
+
+static bool chacha20_use_neon __ro_after_init;
+
+void __init chacha20_fpu_init(void)
+{
+#if defined(CONFIG_ARM64)
+       chacha20_use_neon = elf_hwcap & HWCAP_ASIMD;
+#elif defined(CONFIG_ARM)
+       chacha20_use_neon = elf_hwcap & HWCAP_NEON;
+#endif
+}
+
+static inline bool chacha20_arch(u8 *dst, const u8 *src, const size_t len, 
const u32 key[8], const u32 counter[4], simd_context_t simd_context)
+{
+       if (simd_context != HAVE_FULL_SIMD
+#if defined(ARM_USE_NEON)
+               || !chacha20_use_neon
+#endif
+       )
+               chacha20_arm(dst, src, len, key, counter);
+       else
+               chacha20_neon(dst, src, len, key, counter);
+       return true;
+}
+
+static inline bool hchacha20_arch(u8 *derived_key, const u8 *nonce, const u8 
*key, simd_context_t simd_context) { return false; }
+
+#define HAVE_CHACHA20_ARCH_IMPLEMENTATION
diff --git a/lib/zinc/chacha20/chacha20-arm.S b/lib/zinc/chacha20/chacha20-arm.S
new file mode 100644
index 000000000000..601b4e34283d
--- /dev/null
+++ b/lib/zinc/chacha20/chacha20-arm.S
@@ -0,0 +1,1471 @@
+/* SPDX-License-Identifier: OpenSSL OR (BSD-3-Clause OR GPL-2.0)
+ *
+ * Copyright (C) 2015-2018 Jason A. Donenfeld <ja...@zx2c4.com>. All Rights 
Reserved.
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ */
+
+#include <linux/linkage.h>
+
+.text
+#if defined(__thumb2__) || defined(__clang__)
+.syntax        unified
+#endif
+#if defined(__thumb2__)
+.thumb
+#else
+.code  32
+#endif
+
+#if defined(__thumb2__) || defined(__clang__)
+#define ldrhsb ldrbhs
+#endif
+
+.align 5
+.Lsigma:
+.long  0x61707865,0x3320646e,0x79622d32,0x6b206574     @ endian-neutral
+.Lone:
+.long  1,0,0,0
+.word -1
+
+#if __LINUX_ARM_ARCH__ >= 7 && IS_ENABLED(CONFIG_KERNEL_MODE_NEON)
+.arch  armv7-a
+.fpu   neon
+
+.align 5
+ENTRY(chacha20_neon)
+       ldr             r12,[sp,#0]             @ pull pointer to counter and 
nonce
+       stmdb           sp!,{r0-r2,r4-r11,lr}
+       cmp             r2,#0                   @ len==0?
+#ifdef __thumb2__
+       itt             eq
+#endif
+       addeq           sp,sp,#4*3
+       beq             .Lno_data_neon
+       cmp             r2,#192                 @ test len
+       bls             .Lshort
+.Lchacha20_neon_begin:
+       adr             r14,.Lsigma
+       vstmdb          sp!,{d8-d15}            @ ABI spec says so
+       stmdb           sp!,{r0-r3}
+
+       vld1.32         {q1-q2},[r3]            @ load key
+       ldmia           r3,{r4-r11}             @ load key
+
+       sub             sp,sp,#4*(16+16)
+       vld1.32         {q3},[r12]              @ load counter and nonce
+       add             r12,sp,#4*8
+       ldmia           r14,{r0-r3}             @ load sigma
+       vld1.32         {q0},[r14]!             @ load sigma
+       vld1.32         {q12},[r14]             @ one
+       vst1.32         {q2-q3},[r12]           @ copy 1/2key|counter|nonce
+       vst1.32         {q0-q1},[sp]            @ copy sigma|1/2key
+
+       str             r10,[sp,#4*(16+10)]     @ off-load "rx"
+       str             r11,[sp,#4*(16+11)]     @ off-load "rx"
+       vshl.i32        d26,d24,#1      @ two
+       vstr            d24,[sp,#4*(16+0)]
+       vshl.i32        d28,d24,#2      @ four
+       vstr            d26,[sp,#4*(16+2)]
+       vmov            q4,q0
+       vstr            d28,[sp,#4*(16+4)]
+       vmov            q8,q0
+       vmov            q5,q1
+       vmov            q9,q1
+       b               .Loop_neon_enter
+
+.align 4
+.Loop_neon_outer:
+       ldmia           sp,{r0-r9}              @ load key material
+       cmp             r11,#64*2               @ if len<=64*2
+       bls             .Lbreak_neon            @ switch to integer-only
+       vmov            q4,q0
+       str             r11,[sp,#4*(32+2)]      @ save len
+       vmov            q8,q0
+       str             r12,  [sp,#4*(32+1)]    @ save inp
+       vmov            q5,q1
+       str             r14,  [sp,#4*(32+0)]    @ save out
+       vmov            q9,q1
+.Loop_neon_enter:
+       ldr             r11, [sp,#4*(15)]
+       vadd.i32        q7,q3,q12               @ counter+1
+       ldr             r12,[sp,#4*(12)]        @ modulo-scheduled load
+       vmov            q6,q2
+       ldr             r10, [sp,#4*(13)]
+       vmov            q10,q2
+       ldr             r14,[sp,#4*(14)]
+       vadd.i32        q11,q7,q12              @ counter+2
+       str             r11, [sp,#4*(16+15)]
+       mov             r11,#10
+       add             r12,r12,#3      @ counter+3
+       b               .Loop_neon
+
+.align 4
+.Loop_neon:
+       subs            r11,r11,#1
+       vadd.i32        q0,q0,q1
+       add     r0,r0,r4
+       vadd.i32        q4,q4,q5
+       mov     r12,r12,ror#16
+       vadd.i32        q8,q8,q9
+       add     r1,r1,r5
+       veor    q3,q3,q0
+       mov     r10,r10,ror#16
+       veor    q7,q7,q4
+       eor     r12,r12,r0,ror#16
+       veor    q11,q11,q8
+       eor     r10,r10,r1,ror#16
+       vrev32.16       q3,q3
+       add     r8,r8,r12
+       vrev32.16       q7,q7
+       mov     r4,r4,ror#20
+       vrev32.16       q11,q11
+       add     r9,r9,r10
+       vadd.i32        q2,q2,q3
+       mov     r5,r5,ror#20
+       vadd.i32        q6,q6,q7
+       eor     r4,r4,r8,ror#20
+       vadd.i32        q10,q10,q11
+       eor     r5,r5,r9,ror#20
+       veor    q12,q1,q2
+       add     r0,r0,r4
+       veor    q13,q5,q6
+       mov     r12,r12,ror#24
+       veor    q14,q9,q10
+       add     r1,r1,r5
+       vshr.u32        q1,q12,#20
+       mov     r10,r10,ror#24
+       vshr.u32        q5,q13,#20
+       eor     r12,r12,r0,ror#24
+       vshr.u32        q9,q14,#20
+       eor     r10,r10,r1,ror#24
+       vsli.32 q1,q12,#12
+       add     r8,r8,r12
+       vsli.32 q5,q13,#12
+       mov     r4,r4,ror#25
+       vsli.32 q9,q14,#12
+       add     r9,r9,r10
+       vadd.i32        q0,q0,q1
+       mov     r5,r5,ror#25
+       vadd.i32        q4,q4,q5
+       str     r10,[sp,#4*(16+13)]
+       vadd.i32        q8,q8,q9
+       ldr     r10,[sp,#4*(16+15)]
+       veor    q12,q3,q0
+       eor     r4,r4,r8,ror#25
+       veor    q13,q7,q4
+       eor     r5,r5,r9,ror#25
+       veor    q14,q11,q8
+       str     r8,[sp,#4*(16+8)]
+       vshr.u32        q3,q12,#24
+       ldr     r8,[sp,#4*(16+10)]
+       vshr.u32        q7,q13,#24
+       add     r2,r2,r6
+       vshr.u32        q11,q14,#24
+       mov     r14,r14,ror#16
+       vsli.32 q3,q12,#8
+       str     r9,[sp,#4*(16+9)]
+       vsli.32 q7,q13,#8
+       ldr     r9,[sp,#4*(16+11)]
+       vsli.32 q11,q14,#8
+       add     r3,r3,r7
+       vadd.i32        q2,q2,q3
+       mov     r10,r10,ror#16
+       vadd.i32        q6,q6,q7
+       eor     r14,r14,r2,ror#16
+       vadd.i32        q10,q10,q11
+       eor     r10,r10,r3,ror#16
+       veor    q12,q1,q2
+       add     r8,r8,r14
+       veor    q13,q5,q6
+       mov     r6,r6,ror#20
+       veor    q14,q9,q10
+       add     r9,r9,r10
+       vshr.u32        q1,q12,#25
+       mov     r7,r7,ror#20
+       vshr.u32        q5,q13,#25
+       eor     r6,r6,r8,ror#20
+       vshr.u32        q9,q14,#25
+       eor     r7,r7,r9,ror#20
+       vsli.32 q1,q12,#7
+       add     r2,r2,r6
+       vsli.32 q5,q13,#7
+       mov     r14,r14,ror#24
+       vsli.32 q9,q14,#7
+       add     r3,r3,r7
+       vext.8  q2,q2,q2,#8
+       mov     r10,r10,ror#24
+       vext.8  q6,q6,q6,#8
+       eor     r14,r14,r2,ror#24
+       vext.8  q10,q10,q10,#8
+       eor     r10,r10,r3,ror#24
+       vext.8  q1,q1,q1,#4
+       add     r8,r8,r14
+       vext.8  q5,q5,q5,#4
+       mov     r6,r6,ror#25
+       vext.8  q9,q9,q9,#4
+       add     r9,r9,r10
+       vext.8  q3,q3,q3,#12
+       mov     r7,r7,ror#25
+       vext.8  q7,q7,q7,#12
+       eor     r6,r6,r8,ror#25
+       vext.8  q11,q11,q11,#12
+       eor     r7,r7,r9,ror#25
+       vadd.i32        q0,q0,q1
+       add     r0,r0,r5
+       vadd.i32        q4,q4,q5
+       mov     r10,r10,ror#16
+       vadd.i32        q8,q8,q9
+       add     r1,r1,r6
+       veor    q3,q3,q0
+       mov     r12,r12,ror#16
+       veor    q7,q7,q4
+       eor     r10,r10,r0,ror#16
+       veor    q11,q11,q8
+       eor     r12,r12,r1,ror#16
+       vrev32.16       q3,q3
+       add     r8,r8,r10
+       vrev32.16       q7,q7
+       mov     r5,r5,ror#20
+       vrev32.16       q11,q11
+       add     r9,r9,r12
+       vadd.i32        q2,q2,q3
+       mov     r6,r6,ror#20
+       vadd.i32        q6,q6,q7
+       eor     r5,r5,r8,ror#20
+       vadd.i32        q10,q10,q11
+       eor     r6,r6,r9,ror#20
+       veor    q12,q1,q2
+       add     r0,r0,r5
+       veor    q13,q5,q6
+       mov     r10,r10,ror#24
+       veor    q14,q9,q10
+       add     r1,r1,r6
+       vshr.u32        q1,q12,#20
+       mov     r12,r12,ror#24
+       vshr.u32        q5,q13,#20
+       eor     r10,r10,r0,ror#24
+       vshr.u32        q9,q14,#20
+       eor     r12,r12,r1,ror#24
+       vsli.32 q1,q12,#12
+       add     r8,r8,r10
+       vsli.32 q5,q13,#12
+       mov     r5,r5,ror#25
+       vsli.32 q9,q14,#12
+       str     r10,[sp,#4*(16+15)]
+       vadd.i32        q0,q0,q1
+       ldr     r10,[sp,#4*(16+13)]
+       vadd.i32        q4,q4,q5
+       add     r9,r9,r12
+       vadd.i32        q8,q8,q9
+       mov     r6,r6,ror#25
+       veor    q12,q3,q0
+       eor     r5,r5,r8,ror#25
+       veor    q13,q7,q4
+       eor     r6,r6,r9,ror#25
+       veor    q14,q11,q8
+       str     r8,[sp,#4*(16+10)]
+       vshr.u32        q3,q12,#24
+       ldr     r8,[sp,#4*(16+8)]
+       vshr.u32        q7,q13,#24
+       add     r2,r2,r7
+       vshr.u32        q11,q14,#24
+       mov     r10,r10,ror#16
+       vsli.32 q3,q12,#8
+       str     r9,[sp,#4*(16+11)]
+       vsli.32 q7,q13,#8
+       ldr     r9,[sp,#4*(16+9)]
+       vsli.32 q11,q14,#8
+       add     r3,r3,r4
+       vadd.i32        q2,q2,q3
+       mov     r14,r14,ror#16
+       vadd.i32        q6,q6,q7
+       eor     r10,r10,r2,ror#16
+       vadd.i32        q10,q10,q11
+       eor     r14,r14,r3,ror#16
+       veor    q12,q1,q2
+       add     r8,r8,r10
+       veor    q13,q5,q6
+       mov     r7,r7,ror#20
+       veor    q14,q9,q10
+       add     r9,r9,r14
+       vshr.u32        q1,q12,#25
+       mov     r4,r4,ror#20
+       vshr.u32        q5,q13,#25
+       eor     r7,r7,r8,ror#20
+       vshr.u32        q9,q14,#25
+       eor     r4,r4,r9,ror#20
+       vsli.32 q1,q12,#7
+       add     r2,r2,r7
+       vsli.32 q5,q13,#7
+       mov     r10,r10,ror#24
+       vsli.32 q9,q14,#7
+       add     r3,r3,r4
+       vext.8  q2,q2,q2,#8
+       mov     r14,r14,ror#24
+       vext.8  q6,q6,q6,#8
+       eor     r10,r10,r2,ror#24
+       vext.8  q10,q10,q10,#8
+       eor     r14,r14,r3,ror#24
+       vext.8  q1,q1,q1,#12
+       add     r8,r8,r10
+       vext.8  q5,q5,q5,#12
+       mov     r7,r7,ror#25
+       vext.8  q9,q9,q9,#12
+       add     r9,r9,r14
+       vext.8  q3,q3,q3,#4
+       mov     r4,r4,ror#25
+       vext.8  q7,q7,q7,#4
+       eor     r7,r7,r8,ror#25
+       vext.8  q11,q11,q11,#4
+       eor     r4,r4,r9,ror#25
+       bne             .Loop_neon
+
+       add             r11,sp,#32
+       vld1.32         {q12-q13},[sp]          @ load key material
+       vld1.32         {q14-q15},[r11]
+
+       ldr             r11,[sp,#4*(32+2)]      @ load len
+
+       str             r8, [sp,#4*(16+8)]      @ modulo-scheduled store
+       str             r9, [sp,#4*(16+9)]
+       str             r12,[sp,#4*(16+12)]
+       str             r10, [sp,#4*(16+13)]
+       str             r14,[sp,#4*(16+14)]
+
+       @ at this point we have first half of 512-bit result in
+       @ rx and second half at sp+4*(16+8)
+
+       ldr             r12,[sp,#4*(32+1)]      @ load inp
+       ldr             r14,[sp,#4*(32+0)]      @ load out
+
+       vadd.i32        q0,q0,q12               @ accumulate key material
+       vadd.i32        q4,q4,q12
+       vadd.i32        q8,q8,q12
+       vldr            d24,[sp,#4*(16+0)]      @ one
+
+       vadd.i32        q1,q1,q13
+       vadd.i32        q5,q5,q13
+       vadd.i32        q9,q9,q13
+       vldr            d26,[sp,#4*(16+2)]      @ two
+
+       vadd.i32        q2,q2,q14
+       vadd.i32        q6,q6,q14
+       vadd.i32        q10,q10,q14
+       vadd.i32        d14,d14,d24     @ counter+1
+       vadd.i32        d22,d22,d26     @ counter+2
+
+       vadd.i32        q3,q3,q15
+       vadd.i32        q7,q7,q15
+       vadd.i32        q11,q11,q15
+
+       cmp             r11,#64*4
+       blo             .Ltail_neon
+
+       vld1.8          {q12-q13},[r12]!        @ load input
+        mov            r11,sp
+       vld1.8          {q14-q15},[r12]!
+       veor            q0,q0,q12               @ xor with input
+       veor            q1,q1,q13
+       vld1.8          {q12-q13},[r12]!
+       veor            q2,q2,q14
+       veor            q3,q3,q15
+       vld1.8          {q14-q15},[r12]!
+
+       veor            q4,q4,q12
+        vst1.8         {q0-q1},[r14]!  @ store output
+       veor            q5,q5,q13
+       vld1.8          {q12-q13},[r12]!
+       veor            q6,q6,q14
+        vst1.8         {q2-q3},[r14]!
+       veor            q7,q7,q15
+       vld1.8          {q14-q15},[r12]!
+
+       veor            q8,q8,q12
+        vld1.32        {q0-q1},[r11]!  @ load for next iteration
+        veor           d25,d25,d25
+        vldr           d24,[sp,#4*(16+4)]      @ four
+       veor            q9,q9,q13
+        vld1.32        {q2-q3},[r11]
+       veor            q10,q10,q14
+        vst1.8         {q4-q5},[r14]!
+       veor            q11,q11,q15
+        vst1.8         {q6-q7},[r14]!
+
+       vadd.i32        d6,d6,d24       @ next counter value
+       vldr            d24,[sp,#4*(16+0)]      @ one
+
+       ldmia           sp,{r8-r11}     @ load key material
+       add             r0,r0,r8        @ accumulate key material
+       ldr             r8,[r12],#16            @ load input
+        vst1.8         {q8-q9},[r14]!
+       add             r1,r1,r9
+       ldr             r9,[r12,#-12]
+        vst1.8         {q10-q11},[r14]!
+       add             r2,r2,r10
+       ldr             r10,[r12,#-8]
+       add             r3,r3,r11
+       ldr             r11,[r12,#-4]
+#ifdef __ARMEB__
+       rev             r0,r0
+       rev             r1,r1
+       rev             r2,r2
+       rev             r3,r3
+#endif
+       eor             r0,r0,r8        @ xor with input
+        add            r8,sp,#4*(4)
+       eor             r1,r1,r9
+       str             r0,[r14],#16            @ store output
+       eor             r2,r2,r10
+       str             r1,[r14,#-12]
+       eor             r3,r3,r11
+        ldmia          r8,{r8-r11}     @ load key material
+       str             r2,[r14,#-8]
+       str             r3,[r14,#-4]
+
+       add             r4,r4,r8        @ accumulate key material
+       ldr             r8,[r12],#16            @ load input
+       add             r5,r5,r9
+       ldr             r9,[r12,#-12]
+       add             r6,r6,r10
+       ldr             r10,[r12,#-8]
+       add             r7,r7,r11
+       ldr             r11,[r12,#-4]
+#ifdef __ARMEB__
+       rev             r4,r4
+       rev             r5,r5
+       rev             r6,r6
+       rev             r7,r7
+#endif
+       eor             r4,r4,r8
+        add            r8,sp,#4*(8)
+       eor             r5,r5,r9
+       str             r4,[r14],#16            @ store output
+       eor             r6,r6,r10
+       str             r5,[r14,#-12]
+       eor             r7,r7,r11
+        ldmia          r8,{r8-r11}     @ load key material
+       str             r6,[r14,#-8]
+        add            r0,sp,#4*(16+8)
+       str             r7,[r14,#-4]
+
+       ldmia           r0,{r0-r7}      @ load second half
+
+       add             r0,r0,r8        @ accumulate key material
+       ldr             r8,[r12],#16            @ load input
+       add             r1,r1,r9
+       ldr             r9,[r12,#-12]
+#ifdef __thumb2__
+       it      hi
+#endif
+        strhi          r10,[sp,#4*(16+10)]     @ copy "rx" while at it
+       add             r2,r2,r10
+       ldr             r10,[r12,#-8]
+#ifdef __thumb2__
+       it      hi
+#endif
+        strhi          r11,[sp,#4*(16+11)]     @ copy "rx" while at it
+       add             r3,r3,r11
+       ldr             r11,[r12,#-4]
+#ifdef __ARMEB__
+       rev             r0,r0
+       rev             r1,r1
+       rev             r2,r2
+       rev             r3,r3
+#endif
+       eor             r0,r0,r8
+        add            r8,sp,#4*(12)
+       eor             r1,r1,r9
+       str             r0,[r14],#16            @ store output
+       eor             r2,r2,r10
+       str             r1,[r14,#-12]
+       eor             r3,r3,r11
+        ldmia          r8,{r8-r11}     @ load key material
+       str             r2,[r14,#-8]
+       str             r3,[r14,#-4]
+
+       add             r4,r4,r8        @ accumulate key material
+        add            r8,r8,#4                @ next counter value
+       add             r5,r5,r9
+        str            r8,[sp,#4*(12)] @ save next counter value
+       ldr             r8,[r12],#16            @ load input
+       add             r6,r6,r10
+        add            r4,r4,#3                @ counter+3
+       ldr             r9,[r12,#-12]
+       add             r7,r7,r11
+       ldr             r10,[r12,#-8]
+       ldr             r11,[r12,#-4]
+#ifdef __ARMEB__
+       rev             r4,r4
+       rev             r5,r5
+       rev             r6,r6
+       rev             r7,r7
+#endif
+       eor             r4,r4,r8
+#ifdef __thumb2__
+       it      hi
+#endif
+        ldrhi          r8,[sp,#4*(32+2)]       @ re-load len
+       eor             r5,r5,r9
+       eor             r6,r6,r10
+       str             r4,[r14],#16            @ store output
+       eor             r7,r7,r11
+       str             r5,[r14,#-12]
+        sub            r11,r8,#64*4    @ len-=64*4
+       str             r6,[r14,#-8]
+       str             r7,[r14,#-4]
+       bhi             .Loop_neon_outer
+
+       b               .Ldone_neon
+
+.align 4
+.Lbreak_neon:
+       @ harmonize NEON and integer-only stack frames: load data
+       @ from NEON frame, but save to integer-only one; distance
+       @ between the two is 4*(32+4+16-32)=4*(20).
+
+       str             r11, [sp,#4*(20+32+2)]  @ save len
+        add            r11,sp,#4*(32+4)
+       str             r12,   [sp,#4*(20+32+1)]        @ save inp
+       str             r14,   [sp,#4*(20+32+0)]        @ save out
+
+       ldr             r12,[sp,#4*(16+10)]
+       ldr             r14,[sp,#4*(16+11)]
+        vldmia         r11,{d8-d15}                    @ fulfill ABI 
requirement
+       str             r12,[sp,#4*(20+16+10)]  @ copy "rx"
+       str             r14,[sp,#4*(20+16+11)]  @ copy "rx"
+
+       ldr             r11, [sp,#4*(15)]
+       ldr             r12,[sp,#4*(12)]                @ modulo-scheduled load
+       ldr             r10, [sp,#4*(13)]
+       ldr             r14,[sp,#4*(14)]
+       str             r11, [sp,#4*(20+16+15)]
+       add             r11,sp,#4*(20)
+       vst1.32         {q0-q1},[r11]!          @ copy key
+       add             sp,sp,#4*(20)                   @ switch frame
+       vst1.32         {q2-q3},[r11]
+       mov             r11,#10
+       b               .Loop                           @ go integer-only
+
+.align 4
+.Ltail_neon:
+       cmp             r11,#64*3
+       bhs             .L192_or_more_neon
+       cmp             r11,#64*2
+       bhs             .L128_or_more_neon
+       cmp             r11,#64*1
+       bhs             .L64_or_more_neon
+
+       add             r8,sp,#4*(8)
+       vst1.8          {q0-q1},[sp]
+       add             r10,sp,#4*(0)
+       vst1.8          {q2-q3},[r8]
+       b               .Loop_tail_neon
+
+.align 4
+.L64_or_more_neon:
+       vld1.8          {q12-q13},[r12]!
+       vld1.8          {q14-q15},[r12]!
+       veor            q0,q0,q12
+       veor            q1,q1,q13
+       veor            q2,q2,q14
+       veor            q3,q3,q15
+       vst1.8          {q0-q1},[r14]!
+       vst1.8          {q2-q3},[r14]!
+
+       beq             .Ldone_neon
+
+       add             r8,sp,#4*(8)
+       vst1.8          {q4-q5},[sp]
+       add             r10,sp,#4*(0)
+       vst1.8          {q6-q7},[r8]
+       sub             r11,r11,#64*1   @ len-=64*1
+       b               .Loop_tail_neon
+
+.align 4
+.L128_or_more_neon:
+       vld1.8          {q12-q13},[r12]!
+       vld1.8          {q14-q15},[r12]!
+       veor            q0,q0,q12
+       veor            q1,q1,q13
+       vld1.8          {q12-q13},[r12]!
+       veor            q2,q2,q14
+       veor            q3,q3,q15
+       vld1.8          {q14-q15},[r12]!
+
+       veor            q4,q4,q12
+       veor            q5,q5,q13
+        vst1.8         {q0-q1},[r14]!
+       veor            q6,q6,q14
+        vst1.8         {q2-q3},[r14]!
+       veor            q7,q7,q15
+       vst1.8          {q4-q5},[r14]!
+       vst1.8          {q6-q7},[r14]!
+
+       beq             .Ldone_neon
+
+       add             r8,sp,#4*(8)
+       vst1.8          {q8-q9},[sp]
+       add             r10,sp,#4*(0)
+       vst1.8          {q10-q11},[r8]
+       sub             r11,r11,#64*2   @ len-=64*2
+       b               .Loop_tail_neon
+
+.align 4
+.L192_or_more_neon:
+       vld1.8          {q12-q13},[r12]!
+       vld1.8          {q14-q15},[r12]!
+       veor            q0,q0,q12
+       veor            q1,q1,q13
+       vld1.8          {q12-q13},[r12]!
+       veor            q2,q2,q14
+       veor            q3,q3,q15
+       vld1.8          {q14-q15},[r12]!
+
+       veor            q4,q4,q12
+       veor            q5,q5,q13
+       vld1.8          {q12-q13},[r12]!
+       veor            q6,q6,q14
+        vst1.8         {q0-q1},[r14]!
+       veor            q7,q7,q15
+       vld1.8          {q14-q15},[r12]!
+
+       veor            q8,q8,q12
+        vst1.8         {q2-q3},[r14]!
+       veor            q9,q9,q13
+        vst1.8         {q4-q5},[r14]!
+       veor            q10,q10,q14
+        vst1.8         {q6-q7},[r14]!
+       veor            q11,q11,q15
+       vst1.8          {q8-q9},[r14]!
+       vst1.8          {q10-q11},[r14]!
+
+       beq             .Ldone_neon
+
+       ldmia           sp,{r8-r11}     @ load key material
+       add             r0,r0,r8        @ accumulate key material
+        add            r8,sp,#4*(4)
+       add             r1,r1,r9
+       add             r2,r2,r10
+       add             r3,r3,r11
+        ldmia          r8,{r8-r11}     @ load key material
+
+       add             r4,r4,r8        @ accumulate key material
+        add            r8,sp,#4*(8)
+       add             r5,r5,r9
+       add             r6,r6,r10
+       add             r7,r7,r11
+        ldmia          r8,{r8-r11}     @ load key material
+#ifdef __ARMEB__
+       rev             r0,r0
+       rev             r1,r1
+       rev             r2,r2
+       rev             r3,r3
+       rev             r4,r4
+       rev             r5,r5
+       rev             r6,r6
+       rev             r7,r7
+#endif
+       stmia           sp,{r0-r7}
+        add            r0,sp,#4*(16+8)
+
+       ldmia           r0,{r0-r7}      @ load second half
+
+       add             r0,r0,r8        @ accumulate key material
+        add            r8,sp,#4*(12)
+       add             r1,r1,r9
+       add             r2,r2,r10
+       add             r3,r3,r11
+        ldmia          r8,{r8-r11}     @ load key material
+
+       add             r4,r4,r8        @ accumulate key material
+        add            r8,sp,#4*(8)
+       add             r5,r5,r9
+        add            r4,r4,#3                @ counter+3
+       add             r6,r6,r10
+       add             r7,r7,r11
+        ldr            r11,[sp,#4*(32+2)]      @ re-load len
+#ifdef __ARMEB__
+       rev             r0,r0
+       rev             r1,r1
+       rev             r2,r2
+       rev             r3,r3
+       rev             r4,r4
+       rev             r5,r5
+       rev             r6,r6
+       rev             r7,r7
+#endif
+       stmia           r8,{r0-r7}
+        add            r10,sp,#4*(0)
+        sub            r11,r11,#64*3   @ len-=64*3
+
+.Loop_tail_neon:
+       ldrb            r8,[r10],#1     @ read buffer on stack
+       ldrb            r9,[r12],#1             @ read input
+       subs            r11,r11,#1
+       eor             r8,r8,r9
+       strb            r8,[r14],#1             @ store output
+       bne             .Loop_tail_neon
+
+.Ldone_neon:
+       add             sp,sp,#4*(32+4)
+       vldmia          sp,{d8-d15}
+       add             sp,sp,#4*(16+3)
+.Lno_data_neon:
+       ldmia           sp!,{r4-r11,pc}
+ENDPROC(chacha20_neon)
+#endif
+
+.align 5
+.Lsigma2:
+.long  0x61707865,0x3320646e,0x79622d32,0x6b206574     @ endian-neutral
+.Lone2:
+.long  1,0,0,0
+.word -1
+
+.align 5
+ENTRY(chacha20_arm)
+       ldr     r12,[sp,#0]             @ pull pointer to counter and nonce
+       stmdb   sp!,{r0-r2,r4-r11,lr}
+       cmp     r2,#0                   @ len==0?
+#ifdef __thumb2__
+       itt     eq
+#endif
+       addeq   sp,sp,#4*3
+       beq     .Lno_data_arm
+.Lshort:
+       ldmia   r12,{r4-r7}             @ load counter and nonce
+       sub     sp,sp,#4*(16)           @ off-load area
+#if __LINUX_ARM_ARCH__ < 7 && !defined(__thumb2__)
+       sub     r14,pc,#100             @ .Lsigma2
+#else
+       adr     r14,.Lsigma2            @ .Lsigma2
+#endif
+       stmdb   sp!,{r4-r7}             @ copy counter and nonce
+       ldmia   r3,{r4-r11}             @ load key
+       ldmia   r14,{r0-r3}             @ load sigma
+       stmdb   sp!,{r4-r11}            @ copy key
+       stmdb   sp!,{r0-r3}             @ copy sigma
+       str     r10,[sp,#4*(16+10)]     @ off-load "rx"
+       str     r11,[sp,#4*(16+11)]     @ off-load "rx"
+       b       .Loop_outer_enter
+
+.align 4
+.Loop_outer:
+       ldmia   sp,{r0-r9}              @ load key material
+       str     r11,[sp,#4*(32+2)]      @ save len
+       str     r12,  [sp,#4*(32+1)]    @ save inp
+       str     r14,  [sp,#4*(32+0)]    @ save out
+.Loop_outer_enter:
+       ldr     r11, [sp,#4*(15)]
+       ldr     r12,[sp,#4*(12)]        @ modulo-scheduled load
+       ldr     r10, [sp,#4*(13)]
+       ldr     r14,[sp,#4*(14)]
+       str     r11, [sp,#4*(16+15)]
+       mov     r11,#10
+       b       .Loop
+
+.align 4
+.Loop:
+       subs    r11,r11,#1
+       add     r0,r0,r4
+       mov     r12,r12,ror#16
+       add     r1,r1,r5
+       mov     r10,r10,ror#16
+       eor     r12,r12,r0,ror#16
+       eor     r10,r10,r1,ror#16
+       add     r8,r8,r12
+       mov     r4,r4,ror#20
+       add     r9,r9,r10
+       mov     r5,r5,ror#20
+       eor     r4,r4,r8,ror#20
+       eor     r5,r5,r9,ror#20
+       add     r0,r0,r4
+       mov     r12,r12,ror#24
+       add     r1,r1,r5
+       mov     r10,r10,ror#24
+       eor     r12,r12,r0,ror#24
+       eor     r10,r10,r1,ror#24
+       add     r8,r8,r12
+       mov     r4,r4,ror#25
+       add     r9,r9,r10
+       mov     r5,r5,ror#25
+       str     r10,[sp,#4*(16+13)]
+       ldr     r10,[sp,#4*(16+15)]
+       eor     r4,r4,r8,ror#25
+       eor     r5,r5,r9,ror#25
+       str     r8,[sp,#4*(16+8)]
+       ldr     r8,[sp,#4*(16+10)]
+       add     r2,r2,r6
+       mov     r14,r14,ror#16
+       str     r9,[sp,#4*(16+9)]
+       ldr     r9,[sp,#4*(16+11)]
+       add     r3,r3,r7
+       mov     r10,r10,ror#16
+       eor     r14,r14,r2,ror#16
+       eor     r10,r10,r3,ror#16
+       add     r8,r8,r14
+       mov     r6,r6,ror#20
+       add     r9,r9,r10
+       mov     r7,r7,ror#20
+       eor     r6,r6,r8,ror#20
+       eor     r7,r7,r9,ror#20
+       add     r2,r2,r6
+       mov     r14,r14,ror#24
+       add     r3,r3,r7
+       mov     r10,r10,ror#24
+       eor     r14,r14,r2,ror#24
+       eor     r10,r10,r3,ror#24
+       add     r8,r8,r14
+       mov     r6,r6,ror#25
+       add     r9,r9,r10
+       mov     r7,r7,ror#25
+       eor     r6,r6,r8,ror#25
+       eor     r7,r7,r9,ror#25
+       add     r0,r0,r5
+       mov     r10,r10,ror#16
+       add     r1,r1,r6
+       mov     r12,r12,ror#16
+       eor     r10,r10,r0,ror#16
+       eor     r12,r12,r1,ror#16
+       add     r8,r8,r10
+       mov     r5,r5,ror#20
+       add     r9,r9,r12
+       mov     r6,r6,ror#20
+       eor     r5,r5,r8,ror#20
+       eor     r6,r6,r9,ror#20
+       add     r0,r0,r5
+       mov     r10,r10,ror#24
+       add     r1,r1,r6
+       mov     r12,r12,ror#24
+       eor     r10,r10,r0,ror#24
+       eor     r12,r12,r1,ror#24
+       add     r8,r8,r10
+       mov     r5,r5,ror#25
+       str     r10,[sp,#4*(16+15)]
+       ldr     r10,[sp,#4*(16+13)]
+       add     r9,r9,r12
+       mov     r6,r6,ror#25
+       eor     r5,r5,r8,ror#25
+       eor     r6,r6,r9,ror#25
+       str     r8,[sp,#4*(16+10)]
+       ldr     r8,[sp,#4*(16+8)]
+       add     r2,r2,r7
+       mov     r10,r10,ror#16
+       str     r9,[sp,#4*(16+11)]
+       ldr     r9,[sp,#4*(16+9)]
+       add     r3,r3,r4
+       mov     r14,r14,ror#16
+       eor     r10,r10,r2,ror#16
+       eor     r14,r14,r3,ror#16
+       add     r8,r8,r10
+       mov     r7,r7,ror#20
+       add     r9,r9,r14
+       mov     r4,r4,ror#20
+       eor     r7,r7,r8,ror#20
+       eor     r4,r4,r9,ror#20
+       add     r2,r2,r7
+       mov     r10,r10,ror#24
+       add     r3,r3,r4
+       mov     r14,r14,ror#24
+       eor     r10,r10,r2,ror#24
+       eor     r14,r14,r3,ror#24
+       add     r8,r8,r10
+       mov     r7,r7,ror#25
+       add     r9,r9,r14
+       mov     r4,r4,ror#25
+       eor     r7,r7,r8,ror#25
+       eor     r4,r4,r9,ror#25
+       bne     .Loop
+
+       ldr     r11,[sp,#4*(32+2)]      @ load len
+
+       str     r8, [sp,#4*(16+8)]      @ modulo-scheduled store
+       str     r9, [sp,#4*(16+9)]
+       str     r12,[sp,#4*(16+12)]
+       str     r10, [sp,#4*(16+13)]
+       str     r14,[sp,#4*(16+14)]
+
+       @ at this point we have first half of 512-bit result in
+       @ rx and second half at sp+4*(16+8)
+
+       cmp     r11,#64         @ done yet?
+#ifdef __thumb2__
+       itete   lo
+#endif
+       addlo   r12,sp,#4*(0)           @ shortcut or ...
+       ldrhs   r12,[sp,#4*(32+1)]      @ ... load inp
+       addlo   r14,sp,#4*(0)           @ shortcut or ...
+       ldrhs   r14,[sp,#4*(32+0)]      @ ... load out
+
+       ldr     r8,[sp,#4*(0)]  @ load key material
+       ldr     r9,[sp,#4*(1)]
+
+#if __LINUX_ARM_ARCH__ >= 6 || !defined(__ARMEB__)
+#if __LINUX_ARM_ARCH__ < 7
+       orr     r10,r12,r14
+       tst     r10,#3          @ are input and output aligned?
+       ldr     r10,[sp,#4*(2)]
+       bne     .Lunaligned
+       cmp     r11,#64         @ restore flags
+#else
+       ldr     r10,[sp,#4*(2)]
+#endif
+       ldr     r11,[sp,#4*(3)]
+
+       add     r0,r0,r8        @ accumulate key material
+       add     r1,r1,r9
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r8,[r12],#16            @ load input
+       ldrhs   r9,[r12,#-12]
+
+       add     r2,r2,r10
+       add     r3,r3,r11
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r10,[r12,#-8]
+       ldrhs   r11,[r12,#-4]
+#if __LINUX_ARM_ARCH__ >= 6 && defined(__ARMEB__)
+       rev     r0,r0
+       rev     r1,r1
+       rev     r2,r2
+       rev     r3,r3
+#endif
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r0,r0,r8        @ xor with input
+       eorhs   r1,r1,r9
+        add    r8,sp,#4*(4)
+       str     r0,[r14],#16            @ store output
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r2,r2,r10
+       eorhs   r3,r3,r11
+        ldmia  r8,{r8-r11}     @ load key material
+       str     r1,[r14,#-12]
+       str     r2,[r14,#-8]
+       str     r3,[r14,#-4]
+
+       add     r4,r4,r8        @ accumulate key material
+       add     r5,r5,r9
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r8,[r12],#16            @ load input
+       ldrhs   r9,[r12,#-12]
+       add     r6,r6,r10
+       add     r7,r7,r11
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r10,[r12,#-8]
+       ldrhs   r11,[r12,#-4]
+#if __LINUX_ARM_ARCH__ >= 6 && defined(__ARMEB__)
+       rev     r4,r4
+       rev     r5,r5
+       rev     r6,r6
+       rev     r7,r7
+#endif
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r4,r4,r8
+       eorhs   r5,r5,r9
+        add    r8,sp,#4*(8)
+       str     r4,[r14],#16            @ store output
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r6,r6,r10
+       eorhs   r7,r7,r11
+       str     r5,[r14,#-12]
+        ldmia  r8,{r8-r11}     @ load key material
+       str     r6,[r14,#-8]
+        add    r0,sp,#4*(16+8)
+       str     r7,[r14,#-4]
+
+       ldmia   r0,{r0-r7}      @ load second half
+
+       add     r0,r0,r8        @ accumulate key material
+       add     r1,r1,r9
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r8,[r12],#16            @ load input
+       ldrhs   r9,[r12,#-12]
+#ifdef __thumb2__
+       itt     hi
+#endif
+        strhi  r10,[sp,#4*(16+10)]     @ copy "rx" while at it
+        strhi  r11,[sp,#4*(16+11)]     @ copy "rx" while at it
+       add     r2,r2,r10
+       add     r3,r3,r11
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r10,[r12,#-8]
+       ldrhs   r11,[r12,#-4]
+#if __LINUX_ARM_ARCH__ >= 6 && defined(__ARMEB__)
+       rev     r0,r0
+       rev     r1,r1
+       rev     r2,r2
+       rev     r3,r3
+#endif
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r0,r0,r8
+       eorhs   r1,r1,r9
+        add    r8,sp,#4*(12)
+       str     r0,[r14],#16            @ store output
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r2,r2,r10
+       eorhs   r3,r3,r11
+       str     r1,[r14,#-12]
+        ldmia  r8,{r8-r11}     @ load key material
+       str     r2,[r14,#-8]
+       str     r3,[r14,#-4]
+
+       add     r4,r4,r8        @ accumulate key material
+       add     r5,r5,r9
+#ifdef __thumb2__
+       itt     hi
+#endif
+        addhi  r8,r8,#1                @ next counter value
+        strhi  r8,[sp,#4*(12)] @ save next counter value
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r8,[r12],#16            @ load input
+       ldrhs   r9,[r12,#-12]
+       add     r6,r6,r10
+       add     r7,r7,r11
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhs   r10,[r12,#-8]
+       ldrhs   r11,[r12,#-4]
+#if __LINUX_ARM_ARCH__ >= 6 && defined(__ARMEB__)
+       rev     r4,r4
+       rev     r5,r5
+       rev     r6,r6
+       rev     r7,r7
+#endif
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r4,r4,r8
+       eorhs   r5,r5,r9
+#ifdef __thumb2__
+        it     ne
+#endif
+        ldrne  r8,[sp,#4*(32+2)]       @ re-load len
+#ifdef __thumb2__
+       itt     hs
+#endif
+       eorhs   r6,r6,r10
+       eorhs   r7,r7,r11
+       str     r4,[r14],#16            @ store output
+       str     r5,[r14,#-12]
+#ifdef __thumb2__
+       it      hs
+#endif
+        subhs  r11,r8,#64              @ len-=64
+       str     r6,[r14,#-8]
+       str     r7,[r14,#-4]
+       bhi     .Loop_outer
+
+       beq     .Ldone
+#if __LINUX_ARM_ARCH__ < 7
+       b       .Ltail
+
+.align 4
+.Lunaligned:                           @ unaligned endian-neutral path
+       cmp     r11,#64         @ restore flags
+#endif
+#endif
+#if __LINUX_ARM_ARCH__ < 7
+       ldr     r11,[sp,#4*(3)]
+       add     r0,r0,r8                @ accumulate key material
+       add     r1,r1,r9
+       add     r2,r2,r10
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r8,r8,r8                @ zero or ...
+       ldrhsb  r8,[r12],#16                    @ ... load input
+       eorlo   r9,r9,r9
+       ldrhsb  r9,[r12,#-12]
+
+       add     r3,r3,r11
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r10,r10,r10
+       ldrhsb  r10,[r12,#-8]
+       eorlo   r11,r11,r11
+       ldrhsb  r11,[r12,#-4]
+
+       eor     r0,r8,r0                @ xor with input (or zero)
+       eor     r1,r9,r1
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-15]           @ load more input
+       ldrhsb  r9,[r12,#-11]
+       eor     r2,r10,r2
+        strb   r0,[r14],#16            @ store output
+       eor     r3,r11,r3
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-7]
+       ldrhsb  r11,[r12,#-3]
+        strb   r1,[r14,#-12]
+       eor     r0,r8,r0,lsr#8
+        strb   r2,[r14,#-8]
+       eor     r1,r9,r1,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-14]           @ load more input
+       ldrhsb  r9,[r12,#-10]
+        strb   r3,[r14,#-4]
+       eor     r2,r10,r2,lsr#8
+        strb   r0,[r14,#-15]
+       eor     r3,r11,r3,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-6]
+       ldrhsb  r11,[r12,#-2]
+        strb   r1,[r14,#-11]
+       eor     r0,r8,r0,lsr#8
+        strb   r2,[r14,#-7]
+       eor     r1,r9,r1,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-13]           @ load more input
+       ldrhsb  r9,[r12,#-9]
+        strb   r3,[r14,#-3]
+       eor     r2,r10,r2,lsr#8
+        strb   r0,[r14,#-14]
+       eor     r3,r11,r3,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-5]
+       ldrhsb  r11,[r12,#-1]
+        strb   r1,[r14,#-10]
+        strb   r2,[r14,#-6]
+       eor     r0,r8,r0,lsr#8
+        strb   r3,[r14,#-2]
+       eor     r1,r9,r1,lsr#8
+        strb   r0,[r14,#-13]
+       eor     r2,r10,r2,lsr#8
+        strb   r1,[r14,#-9]
+       eor     r3,r11,r3,lsr#8
+        strb   r2,[r14,#-5]
+        strb   r3,[r14,#-1]
+       add     r8,sp,#4*(4+0)
+       ldmia   r8,{r8-r11}             @ load key material
+       add     r0,sp,#4*(16+8)
+       add     r4,r4,r8                @ accumulate key material
+       add     r5,r5,r9
+       add     r6,r6,r10
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r8,r8,r8                @ zero or ...
+       ldrhsb  r8,[r12],#16                    @ ... load input
+       eorlo   r9,r9,r9
+       ldrhsb  r9,[r12,#-12]
+
+       add     r7,r7,r11
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r10,r10,r10
+       ldrhsb  r10,[r12,#-8]
+       eorlo   r11,r11,r11
+       ldrhsb  r11,[r12,#-4]
+
+       eor     r4,r8,r4                @ xor with input (or zero)
+       eor     r5,r9,r5
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-15]           @ load more input
+       ldrhsb  r9,[r12,#-11]
+       eor     r6,r10,r6
+        strb   r4,[r14],#16            @ store output
+       eor     r7,r11,r7
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-7]
+       ldrhsb  r11,[r12,#-3]
+        strb   r5,[r14,#-12]
+       eor     r4,r8,r4,lsr#8
+        strb   r6,[r14,#-8]
+       eor     r5,r9,r5,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-14]           @ load more input
+       ldrhsb  r9,[r12,#-10]
+        strb   r7,[r14,#-4]
+       eor     r6,r10,r6,lsr#8
+        strb   r4,[r14,#-15]
+       eor     r7,r11,r7,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-6]
+       ldrhsb  r11,[r12,#-2]
+        strb   r5,[r14,#-11]
+       eor     r4,r8,r4,lsr#8
+        strb   r6,[r14,#-7]
+       eor     r5,r9,r5,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-13]           @ load more input
+       ldrhsb  r9,[r12,#-9]
+        strb   r7,[r14,#-3]
+       eor     r6,r10,r6,lsr#8
+        strb   r4,[r14,#-14]
+       eor     r7,r11,r7,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-5]
+       ldrhsb  r11,[r12,#-1]
+        strb   r5,[r14,#-10]
+        strb   r6,[r14,#-6]
+       eor     r4,r8,r4,lsr#8
+        strb   r7,[r14,#-2]
+       eor     r5,r9,r5,lsr#8
+        strb   r4,[r14,#-13]
+       eor     r6,r10,r6,lsr#8
+        strb   r5,[r14,#-9]
+       eor     r7,r11,r7,lsr#8
+        strb   r6,[r14,#-5]
+        strb   r7,[r14,#-1]
+       add     r8,sp,#4*(4+4)
+       ldmia   r8,{r8-r11}             @ load key material
+       ldmia   r0,{r0-r7}              @ load second half
+#ifdef __thumb2__
+       itt     hi
+#endif
+       strhi   r10,[sp,#4*(16+10)]             @ copy "rx"
+       strhi   r11,[sp,#4*(16+11)]             @ copy "rx"
+       add     r0,r0,r8                @ accumulate key material
+       add     r1,r1,r9
+       add     r2,r2,r10
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r8,r8,r8                @ zero or ...
+       ldrhsb  r8,[r12],#16                    @ ... load input
+       eorlo   r9,r9,r9
+       ldrhsb  r9,[r12,#-12]
+
+       add     r3,r3,r11
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r10,r10,r10
+       ldrhsb  r10,[r12,#-8]
+       eorlo   r11,r11,r11
+       ldrhsb  r11,[r12,#-4]
+
+       eor     r0,r8,r0                @ xor with input (or zero)
+       eor     r1,r9,r1
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-15]           @ load more input
+       ldrhsb  r9,[r12,#-11]
+       eor     r2,r10,r2
+        strb   r0,[r14],#16            @ store output
+       eor     r3,r11,r3
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-7]
+       ldrhsb  r11,[r12,#-3]
+        strb   r1,[r14,#-12]
+       eor     r0,r8,r0,lsr#8
+        strb   r2,[r14,#-8]
+       eor     r1,r9,r1,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-14]           @ load more input
+       ldrhsb  r9,[r12,#-10]
+        strb   r3,[r14,#-4]
+       eor     r2,r10,r2,lsr#8
+        strb   r0,[r14,#-15]
+       eor     r3,r11,r3,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-6]
+       ldrhsb  r11,[r12,#-2]
+        strb   r1,[r14,#-11]
+       eor     r0,r8,r0,lsr#8
+        strb   r2,[r14,#-7]
+       eor     r1,r9,r1,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-13]           @ load more input
+       ldrhsb  r9,[r12,#-9]
+        strb   r3,[r14,#-3]
+       eor     r2,r10,r2,lsr#8
+        strb   r0,[r14,#-14]
+       eor     r3,r11,r3,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-5]
+       ldrhsb  r11,[r12,#-1]
+        strb   r1,[r14,#-10]
+        strb   r2,[r14,#-6]
+       eor     r0,r8,r0,lsr#8
+        strb   r3,[r14,#-2]
+       eor     r1,r9,r1,lsr#8
+        strb   r0,[r14,#-13]
+       eor     r2,r10,r2,lsr#8
+        strb   r1,[r14,#-9]
+       eor     r3,r11,r3,lsr#8
+        strb   r2,[r14,#-5]
+        strb   r3,[r14,#-1]
+       add     r8,sp,#4*(4+8)
+       ldmia   r8,{r8-r11}             @ load key material
+       add     r4,r4,r8                @ accumulate key material
+#ifdef __thumb2__
+       itt     hi
+#endif
+       addhi   r8,r8,#1                        @ next counter value
+       strhi   r8,[sp,#4*(12)]         @ save next counter value
+       add     r5,r5,r9
+       add     r6,r6,r10
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r8,r8,r8                @ zero or ...
+       ldrhsb  r8,[r12],#16                    @ ... load input
+       eorlo   r9,r9,r9
+       ldrhsb  r9,[r12,#-12]
+
+       add     r7,r7,r11
+#ifdef __thumb2__
+       itete   lo
+#endif
+       eorlo   r10,r10,r10
+       ldrhsb  r10,[r12,#-8]
+       eorlo   r11,r11,r11
+       ldrhsb  r11,[r12,#-4]
+
+       eor     r4,r8,r4                @ xor with input (or zero)
+       eor     r5,r9,r5
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-15]           @ load more input
+       ldrhsb  r9,[r12,#-11]
+       eor     r6,r10,r6
+        strb   r4,[r14],#16            @ store output
+       eor     r7,r11,r7
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-7]
+       ldrhsb  r11,[r12,#-3]
+        strb   r5,[r14,#-12]
+       eor     r4,r8,r4,lsr#8
+        strb   r6,[r14,#-8]
+       eor     r5,r9,r5,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-14]           @ load more input
+       ldrhsb  r9,[r12,#-10]
+        strb   r7,[r14,#-4]
+       eor     r6,r10,r6,lsr#8
+        strb   r4,[r14,#-15]
+       eor     r7,r11,r7,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-6]
+       ldrhsb  r11,[r12,#-2]
+        strb   r5,[r14,#-11]
+       eor     r4,r8,r4,lsr#8
+        strb   r6,[r14,#-7]
+       eor     r5,r9,r5,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r8,[r12,#-13]           @ load more input
+       ldrhsb  r9,[r12,#-9]
+        strb   r7,[r14,#-3]
+       eor     r6,r10,r6,lsr#8
+        strb   r4,[r14,#-14]
+       eor     r7,r11,r7,lsr#8
+#ifdef __thumb2__
+       itt     hs
+#endif
+       ldrhsb  r10,[r12,#-5]
+       ldrhsb  r11,[r12,#-1]
+        strb   r5,[r14,#-10]
+        strb   r6,[r14,#-6]
+       eor     r4,r8,r4,lsr#8
+        strb   r7,[r14,#-2]
+       eor     r5,r9,r5,lsr#8
+        strb   r4,[r14,#-13]
+       eor     r6,r10,r6,lsr#8
+        strb   r5,[r14,#-9]
+       eor     r7,r11,r7,lsr#8
+        strb   r6,[r14,#-5]
+        strb   r7,[r14,#-1]
+#ifdef __thumb2__
+       it      ne
+#endif
+       ldrne   r8,[sp,#4*(32+2)]               @ re-load len
+#ifdef __thumb2__
+       it      hs
+#endif
+       subhs   r11,r8,#64                      @ len-=64
+       bhi     .Loop_outer
+
+       beq     .Ldone
+#endif
+
+.Ltail:
+       ldr     r12,[sp,#4*(32+1)]      @ load inp
+       add     r9,sp,#4*(0)
+       ldr     r14,[sp,#4*(32+0)]      @ load out
+
+.Loop_tail:
+       ldrb    r10,[r9],#1     @ read buffer on stack
+       ldrb    r11,[r12],#1            @ read input
+       subs    r8,r8,#1
+       eor     r11,r11,r10
+       strb    r11,[r14],#1            @ store output
+       bne     .Loop_tail
+
+.Ldone:
+       add     sp,sp,#4*(32+3)
+.Lno_data_arm:
+       ldmia   sp!,{r4-r11,pc}
+ENDPROC(chacha20_arm)
diff --git a/lib/zinc/chacha20/chacha20-arm64.S 
b/lib/zinc/chacha20/chacha20-arm64.S
new file mode 100644
index 000000000000..c3d1243161a8
--- /dev/null
+++ b/lib/zinc/chacha20/chacha20-arm64.S
@@ -0,0 +1,1940 @@
+/* SPDX-License-Identifier: OpenSSL OR (BSD-3-Clause OR GPL-2.0)
+ *
+ * Copyright (C) 2015-2018 Jason A. Donenfeld <ja...@zx2c4.com>. All Rights 
Reserved.
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ */
+
+#include <linux/linkage.h>
+
+.text
+.align 5
+.Lsigma:
+.quad  0x3320646e61707865,0x6b20657479622d32           // endian-neutral
+.Lone:
+.long  1,0,0,0
+
+.align 5
+ENTRY(chacha20_arm)
+       cbz     x2,.Labort
+.Lshort:
+       stp     x29,x30,[sp,#-96]!
+       add     x29,sp,#0
+
+       adr     x5,.Lsigma
+       stp     x19,x20,[sp,#16]
+       stp     x21,x22,[sp,#32]
+       stp     x23,x24,[sp,#48]
+       stp     x25,x26,[sp,#64]
+       stp     x27,x28,[sp,#80]
+       sub     sp,sp,#64
+
+       ldp     x22,x23,[x5]            // load sigma
+       ldp     x24,x25,[x3]            // load key
+       ldp     x26,x27,[x3,#16]
+       ldp     x28,x30,[x4]            // load counter
+#ifdef __ARMEB__
+       ror     x24,x24,#32
+       ror     x25,x25,#32
+       ror     x26,x26,#32
+       ror     x27,x27,#32
+       ror     x28,x28,#32
+       ror     x30,x30,#32
+#endif
+
+.Loop_outer:
+       mov     w5,w22                  // unpack key block
+       lsr     x6,x22,#32
+       mov     w7,w23
+       lsr     x8,x23,#32
+       mov     w9,w24
+       lsr     x10,x24,#32
+       mov     w11,w25
+       lsr     x12,x25,#32
+       mov     w13,w26
+       lsr     x14,x26,#32
+       mov     w15,w27
+       lsr     x16,x27,#32
+       mov     w17,w28
+       lsr     x19,x28,#32
+       mov     w20,w30
+       lsr     x21,x30,#32
+
+       mov     x4,#10
+       subs    x2,x2,#64
+.Loop:
+       sub     x4,x4,#1
+       add     w5,w5,w9
+       add     w6,w6,w10
+       add     w7,w7,w11
+       add     w8,w8,w12
+       eor     w17,w17,w5
+       eor     w19,w19,w6
+       eor     w20,w20,w7
+       eor     w21,w21,w8
+       ror     w17,w17,#16
+       ror     w19,w19,#16
+       ror     w20,w20,#16
+       ror     w21,w21,#16
+       add     w13,w13,w17
+       add     w14,w14,w19
+       add     w15,w15,w20
+       add     w16,w16,w21
+       eor     w9,w9,w13
+       eor     w10,w10,w14
+       eor     w11,w11,w15
+       eor     w12,w12,w16
+       ror     w9,w9,#20
+       ror     w10,w10,#20
+       ror     w11,w11,#20
+       ror     w12,w12,#20
+       add     w5,w5,w9
+       add     w6,w6,w10
+       add     w7,w7,w11
+       add     w8,w8,w12
+       eor     w17,w17,w5
+       eor     w19,w19,w6
+       eor     w20,w20,w7
+       eor     w21,w21,w8
+       ror     w17,w17,#24
+       ror     w19,w19,#24
+       ror     w20,w20,#24
+       ror     w21,w21,#24
+       add     w13,w13,w17
+       add     w14,w14,w19
+       add     w15,w15,w20
+       add     w16,w16,w21
+       eor     w9,w9,w13
+       eor     w10,w10,w14
+       eor     w11,w11,w15
+       eor     w12,w12,w16
+       ror     w9,w9,#25
+       ror     w10,w10,#25
+       ror     w11,w11,#25
+       ror     w12,w12,#25
+       add     w5,w5,w10
+       add     w6,w6,w11
+       add     w7,w7,w12
+       add     w8,w8,w9
+       eor     w21,w21,w5
+       eor     w17,w17,w6
+       eor     w19,w19,w7
+       eor     w20,w20,w8
+       ror     w21,w21,#16
+       ror     w17,w17,#16
+       ror     w19,w19,#16
+       ror     w20,w20,#16
+       add     w15,w15,w21
+       add     w16,w16,w17
+       add     w13,w13,w19
+       add     w14,w14,w20
+       eor     w10,w10,w15
+       eor     w11,w11,w16
+       eor     w12,w12,w13
+       eor     w9,w9,w14
+       ror     w10,w10,#20
+       ror     w11,w11,#20
+       ror     w12,w12,#20
+       ror     w9,w9,#20
+       add     w5,w5,w10
+       add     w6,w6,w11
+       add     w7,w7,w12
+       add     w8,w8,w9
+       eor     w21,w21,w5
+       eor     w17,w17,w6
+       eor     w19,w19,w7
+       eor     w20,w20,w8
+       ror     w21,w21,#24
+       ror     w17,w17,#24
+       ror     w19,w19,#24
+       ror     w20,w20,#24
+       add     w15,w15,w21
+       add     w16,w16,w17
+       add     w13,w13,w19
+       add     w14,w14,w20
+       eor     w10,w10,w15
+       eor     w11,w11,w16
+       eor     w12,w12,w13
+       eor     w9,w9,w14
+       ror     w10,w10,#25
+       ror     w11,w11,#25
+       ror     w12,w12,#25
+       ror     w9,w9,#25
+       cbnz    x4,.Loop
+
+       add     w5,w5,w22               // accumulate key block
+       add     x6,x6,x22,lsr#32
+       add     w7,w7,w23
+       add     x8,x8,x23,lsr#32
+       add     w9,w9,w24
+       add     x10,x10,x24,lsr#32
+       add     w11,w11,w25
+       add     x12,x12,x25,lsr#32
+       add     w13,w13,w26
+       add     x14,x14,x26,lsr#32
+       add     w15,w15,w27
+       add     x16,x16,x27,lsr#32
+       add     w17,w17,w28
+       add     x19,x19,x28,lsr#32
+       add     w20,w20,w30
+       add     x21,x21,x30,lsr#32
+
+       b.lo    .Ltail
+
+       add     x5,x5,x6,lsl#32 // pack
+       add     x7,x7,x8,lsl#32
+       ldp     x6,x8,[x1,#0]           // load input
+       add     x9,x9,x10,lsl#32
+       add     x11,x11,x12,lsl#32
+       ldp     x10,x12,[x1,#16]
+       add     x13,x13,x14,lsl#32
+       add     x15,x15,x16,lsl#32
+       ldp     x14,x16,[x1,#32]
+       add     x17,x17,x19,lsl#32
+       add     x20,x20,x21,lsl#32
+       ldp     x19,x21,[x1,#48]
+       add     x1,x1,#64
+#ifdef __ARMEB__
+       rev     x5,x5
+       rev     x7,x7
+       rev     x9,x9
+       rev     x11,x11
+       rev     x13,x13
+       rev     x15,x15
+       rev     x17,x17
+       rev     x20,x20
+#endif
+       eor     x5,x5,x6
+       eor     x7,x7,x8
+       eor     x9,x9,x10
+       eor     x11,x11,x12
+       eor     x13,x13,x14
+       eor     x15,x15,x16
+       eor     x17,x17,x19
+       eor     x20,x20,x21
+
+       stp     x5,x7,[x0,#0]           // store output
+       add     x28,x28,#1                      // increment counter
+       stp     x9,x11,[x0,#16]
+       stp     x13,x15,[x0,#32]
+       stp     x17,x20,[x0,#48]
+       add     x0,x0,#64
+
+       b.hi    .Loop_outer
+
+       ldp     x19,x20,[x29,#16]
+       add     sp,sp,#64
+       ldp     x21,x22,[x29,#32]
+       ldp     x23,x24,[x29,#48]
+       ldp     x25,x26,[x29,#64]
+       ldp     x27,x28,[x29,#80]
+       ldp     x29,x30,[sp],#96
+.Labort:
+       ret
+
+.align 4
+.Ltail:
+       add     x2,x2,#64
+.Less_than_64:
+       sub     x0,x0,#1
+       add     x1,x1,x2
+       add     x0,x0,x2
+       add     x4,sp,x2
+       neg     x2,x2
+
+       add     x5,x5,x6,lsl#32 // pack
+       add     x7,x7,x8,lsl#32
+       add     x9,x9,x10,lsl#32
+       add     x11,x11,x12,lsl#32
+       add     x13,x13,x14,lsl#32
+       add     x15,x15,x16,lsl#32
+       add     x17,x17,x19,lsl#32
+       add     x20,x20,x21,lsl#32
+#ifdef __ARMEB__
+       rev     x5,x5
+       rev     x7,x7
+       rev     x9,x9
+       rev     x11,x11
+       rev     x13,x13
+       rev     x15,x15
+       rev     x17,x17
+       rev     x20,x20
+#endif
+       stp     x5,x7,[sp,#0]
+       stp     x9,x11,[sp,#16]
+       stp     x13,x15,[sp,#32]
+       stp     x17,x20,[sp,#48]
+
+.Loop_tail:
+       ldrb    w10,[x1,x2]
+       ldrb    w11,[x4,x2]
+       add     x2,x2,#1
+       eor     w10,w10,w11
+       strb    w10,[x0,x2]
+       cbnz    x2,.Loop_tail
+
+       stp     xzr,xzr,[sp,#0]
+       stp     xzr,xzr,[sp,#16]
+       stp     xzr,xzr,[sp,#32]
+       stp     xzr,xzr,[sp,#48]
+
+       ldp     x19,x20,[x29,#16]
+       add     sp,sp,#64
+       ldp     x21,x22,[x29,#32]
+       ldp     x23,x24,[x29,#48]
+       ldp     x25,x26,[x29,#64]
+       ldp     x27,x28,[x29,#80]
+       ldp     x29,x30,[sp],#96
+       ret
+ENDPROC(chacha20_arm)
+
+.align 5
+ENTRY(chacha20_neon)
+       cbz     x2,.Labort_neon
+       cmp     x2,#192
+       b.lo    .Lshort
+
+       stp     x29,x30,[sp,#-96]!
+       add     x29,sp,#0
+
+       adr     x5,.Lsigma
+       stp     x19,x20,[sp,#16]
+       stp     x21,x22,[sp,#32]
+       stp     x23,x24,[sp,#48]
+       stp     x25,x26,[sp,#64]
+       stp     x27,x28,[sp,#80]
+       cmp     x2,#512
+       b.hs    .L512_or_more_neon
+
+       sub     sp,sp,#64
+
+       ldp     x22,x23,[x5]            // load sigma
+       ld1     {v24.4s},[x5],#16
+       ldp     x24,x25,[x3]            // load key
+       ldp     x26,x27,[x3,#16]
+       ld1     {v25.4s,v26.4s},[x3]
+       ldp     x28,x30,[x4]            // load counter
+       ld1     {v27.4s},[x4]
+       ld1     {v31.4s},[x5]
+#ifdef __ARMEB__
+       rev64   v24.4s,v24.4s
+       ror     x24,x24,#32
+       ror     x25,x25,#32
+       ror     x26,x26,#32
+       ror     x27,x27,#32
+       ror     x28,x28,#32
+       ror     x30,x30,#32
+#endif
+       add     v27.4s,v27.4s,v31.4s            // += 1
+       add     v28.4s,v27.4s,v31.4s
+       add     v29.4s,v28.4s,v31.4s
+       shl     v31.4s,v31.4s,#2                        // 1 -> 4
+
+.Loop_outer_neon:
+       mov     w5,w22                  // unpack key block
+       lsr     x6,x22,#32
+       mov     v0.16b,v24.16b
+       mov     w7,w23
+       lsr     x8,x23,#32
+       mov     v4.16b,v24.16b
+       mov     w9,w24
+       lsr     x10,x24,#32
+       mov     v16.16b,v24.16b
+       mov     w11,w25
+       mov     v1.16b,v25.16b
+       lsr     x12,x25,#32
+       mov     v5.16b,v25.16b
+       mov     w13,w26
+       mov     v17.16b,v25.16b
+       lsr     x14,x26,#32
+       mov     v3.16b,v27.16b
+       mov     w15,w27
+       mov     v7.16b,v28.16b
+       lsr     x16,x27,#32
+       mov     v19.16b,v29.16b
+       mov     w17,w28
+       mov     v2.16b,v26.16b
+       lsr     x19,x28,#32
+       mov     v6.16b,v26.16b
+       mov     w20,w30
+       mov     v18.16b,v26.16b
+       lsr     x21,x30,#32
+
+       mov     x4,#10
+       subs    x2,x2,#256
+.Loop_neon:
+       sub     x4,x4,#1
+       add     v0.4s,v0.4s,v1.4s
+       add     w5,w5,w9
+       add     v4.4s,v4.4s,v5.4s
+       add     w6,w6,w10
+       add     v16.4s,v16.4s,v17.4s
+       add     w7,w7,w11
+       eor     v3.16b,v3.16b,v0.16b
+       add     w8,w8,w12
+       eor     v7.16b,v7.16b,v4.16b
+       eor     w17,w17,w5
+       eor     v19.16b,v19.16b,v16.16b
+       eor     w19,w19,w6
+       rev32   v3.8h,v3.8h
+       eor     w20,w20,w7
+       rev32   v7.8h,v7.8h
+       eor     w21,w21,w8
+       rev32   v19.8h,v19.8h
+       ror     w17,w17,#16
+       add     v2.4s,v2.4s,v3.4s
+       ror     w19,w19,#16
+       add     v6.4s,v6.4s,v7.4s
+       ror     w20,w20,#16
+       add     v18.4s,v18.4s,v19.4s
+       ror     w21,w21,#16
+       eor     v20.16b,v1.16b,v2.16b
+       add     w13,w13,w17
+       eor     v21.16b,v5.16b,v6.16b
+       add     w14,w14,w19
+       eor     v22.16b,v17.16b,v18.16b
+       add     w15,w15,w20
+       ushr    v1.4s,v20.4s,#20
+       add     w16,w16,w21
+       ushr    v5.4s,v21.4s,#20
+       eor     w9,w9,w13
+       ushr    v17.4s,v22.4s,#20
+       eor     w10,w10,w14
+       sli     v1.4s,v20.4s,#12
+       eor     w11,w11,w15
+       sli     v5.4s,v21.4s,#12
+       eor     w12,w12,w16
+       sli     v17.4s,v22.4s,#12
+       ror     w9,w9,#20
+       add     v0.4s,v0.4s,v1.4s
+       ror     w10,w10,#20
+       add     v4.4s,v4.4s,v5.4s
+       ror     w11,w11,#20
+       add     v16.4s,v16.4s,v17.4s
+       ror     w12,w12,#20
+       eor     v20.16b,v3.16b,v0.16b
+       add     w5,w5,w9
+       eor     v21.16b,v7.16b,v4.16b
+       add     w6,w6,w10
+       eor     v22.16b,v19.16b,v16.16b
+       add     w7,w7,w11
+       ushr    v3.4s,v20.4s,#24
+       add     w8,w8,w12
+       ushr    v7.4s,v21.4s,#24
+       eor     w17,w17,w5
+       ushr    v19.4s,v22.4s,#24
+       eor     w19,w19,w6
+       sli     v3.4s,v20.4s,#8
+       eor     w20,w20,w7
+       sli     v7.4s,v21.4s,#8
+       eor     w21,w21,w8
+       sli     v19.4s,v22.4s,#8
+       ror     w17,w17,#24
+       add     v2.4s,v2.4s,v3.4s
+       ror     w19,w19,#24
+       add     v6.4s,v6.4s,v7.4s
+       ror     w20,w20,#24
+       add     v18.4s,v18.4s,v19.4s
+       ror     w21,w21,#24
+       eor     v20.16b,v1.16b,v2.16b
+       add     w13,w13,w17
+       eor     v21.16b,v5.16b,v6.16b
+       add     w14,w14,w19
+       eor     v22.16b,v17.16b,v18.16b
+       add     w15,w15,w20
+       ushr    v1.4s,v20.4s,#25
+       add     w16,w16,w21
+       ushr    v5.4s,v21.4s,#25
+       eor     w9,w9,w13
+       ushr    v17.4s,v22.4s,#25
+       eor     w10,w10,w14
+       sli     v1.4s,v20.4s,#7
+       eor     w11,w11,w15
+       sli     v5.4s,v21.4s,#7
+       eor     w12,w12,w16
+       sli     v17.4s,v22.4s,#7
+       ror     w9,w9,#25
+       ext     v2.16b,v2.16b,v2.16b,#8
+       ror     w10,w10,#25
+       ext     v6.16b,v6.16b,v6.16b,#8
+       ror     w11,w11,#25
+       ext     v18.16b,v18.16b,v18.16b,#8
+       ror     w12,w12,#25
+       ext     v3.16b,v3.16b,v3.16b,#12
+       ext     v7.16b,v7.16b,v7.16b,#12
+       ext     v19.16b,v19.16b,v19.16b,#12
+       ext     v1.16b,v1.16b,v1.16b,#4
+       ext     v5.16b,v5.16b,v5.16b,#4
+       ext     v17.16b,v17.16b,v17.16b,#4
+       add     v0.4s,v0.4s,v1.4s
+       add     w5,w5,w10
+       add     v4.4s,v4.4s,v5.4s
+       add     w6,w6,w11
+       add     v16.4s,v16.4s,v17.4s
+       add     w7,w7,w12
+       eor     v3.16b,v3.16b,v0.16b
+       add     w8,w8,w9
+       eor     v7.16b,v7.16b,v4.16b
+       eor     w21,w21,w5
+       eor     v19.16b,v19.16b,v16.16b
+       eor     w17,w17,w6
+       rev32   v3.8h,v3.8h
+       eor     w19,w19,w7
+       rev32   v7.8h,v7.8h
+       eor     w20,w20,w8
+       rev32   v19.8h,v19.8h
+       ror     w21,w21,#16
+       add     v2.4s,v2.4s,v3.4s
+       ror     w17,w17,#16
+       add     v6.4s,v6.4s,v7.4s
+       ror     w19,w19,#16
+       add     v18.4s,v18.4s,v19.4s
+       ror     w20,w20,#16
+       eor     v20.16b,v1.16b,v2.16b
+       add     w15,w15,w21
+       eor     v21.16b,v5.16b,v6.16b
+       add     w16,w16,w17
+       eor     v22.16b,v17.16b,v18.16b
+       add     w13,w13,w19
+       ushr    v1.4s,v20.4s,#20
+       add     w14,w14,w20
+       ushr    v5.4s,v21.4s,#20
+       eor     w10,w10,w15
+       ushr    v17.4s,v22.4s,#20
+       eor     w11,w11,w16
+       sli     v1.4s,v20.4s,#12
+       eor     w12,w12,w13
+       sli     v5.4s,v21.4s,#12
+       eor     w9,w9,w14
+       sli     v17.4s,v22.4s,#12
+       ror     w10,w10,#20
+       add     v0.4s,v0.4s,v1.4s
+       ror     w11,w11,#20
+       add     v4.4s,v4.4s,v5.4s
+       ror     w12,w12,#20
+       add     v16.4s,v16.4s,v17.4s
+       ror     w9,w9,#20
+       eor     v20.16b,v3.16b,v0.16b
+       add     w5,w5,w10
+       eor     v21.16b,v7.16b,v4.16b
+       add     w6,w6,w11
+       eor     v22.16b,v19.16b,v16.16b
+       add     w7,w7,w12
+       ushr    v3.4s,v20.4s,#24
+       add     w8,w8,w9
+       ushr    v7.4s,v21.4s,#24
+       eor     w21,w21,w5
+       ushr    v19.4s,v22.4s,#24
+       eor     w17,w17,w6
+       sli     v3.4s,v20.4s,#8
+       eor     w19,w19,w7
+       sli     v7.4s,v21.4s,#8
+       eor     w20,w20,w8
+       sli     v19.4s,v22.4s,#8
+       ror     w21,w21,#24
+       add     v2.4s,v2.4s,v3.4s
+       ror     w17,w17,#24
+       add     v6.4s,v6.4s,v7.4s
+       ror     w19,w19,#24
+       add     v18.4s,v18.4s,v19.4s
+       ror     w20,w20,#24
+       eor     v20.16b,v1.16b,v2.16b
+       add     w15,w15,w21
+       eor     v21.16b,v5.16b,v6.16b
+       add     w16,w16,w17
+       eor     v22.16b,v17.16b,v18.16b
+       add     w13,w13,w19
+       ushr    v1.4s,v20.4s,#25
+       add     w14,w14,w20
+       ushr    v5.4s,v21.4s,#25
+       eor     w10,w10,w15
+       ushr    v17.4s,v22.4s,#25
+       eor     w11,w11,w16
+       sli     v1.4s,v20.4s,#7
+       eor     w12,w12,w13
+       sli     v5.4s,v21.4s,#7
+       eor     w9,w9,w14
+       sli     v17.4s,v22.4s,#7
+       ror     w10,w10,#25
+       ext     v2.16b,v2.16b,v2.16b,#8
+       ror     w11,w11,#25
+       ext     v6.16b,v6.16b,v6.16b,#8
+       ror     w12,w12,#25
+       ext     v18.16b,v18.16b,v18.16b,#8
+       ror     w9,w9,#25
+       ext     v3.16b,v3.16b,v3.16b,#4
+       ext     v7.16b,v7.16b,v7.16b,#4
+       ext     v19.16b,v19.16b,v19.16b,#4
+       ext     v1.16b,v1.16b,v1.16b,#12
+       ext     v5.16b,v5.16b,v5.16b,#12
+       ext     v17.16b,v17.16b,v17.16b,#12
+       cbnz    x4,.Loop_neon
+
+       add     w5,w5,w22               // accumulate key block
+       add     v0.4s,v0.4s,v24.4s
+       add     x6,x6,x22,lsr#32
+       add     v4.4s,v4.4s,v24.4s
+       add     w7,w7,w23
+       add     v16.4s,v16.4s,v24.4s
+       add     x8,x8,x23,lsr#32
+       add     v2.4s,v2.4s,v26.4s
+       add     w9,w9,w24
+       add     v6.4s,v6.4s,v26.4s
+       add     x10,x10,x24,lsr#32
+       add     v18.4s,v18.4s,v26.4s
+       add     w11,w11,w25
+       add     v3.4s,v3.4s,v27.4s
+       add     x12,x12,x25,lsr#32
+       add     w13,w13,w26
+       add     v7.4s,v7.4s,v28.4s
+       add     x14,x14,x26,lsr#32
+       add     w15,w15,w27
+       add     v19.4s,v19.4s,v29.4s
+       add     x16,x16,x27,lsr#32
+       add     w17,w17,w28
+       add     v1.4s,v1.4s,v25.4s
+       add     x19,x19,x28,lsr#32
+       add     w20,w20,w30
+       add     v5.4s,v5.4s,v25.4s
+       add     x21,x21,x30,lsr#32
+       add     v17.4s,v17.4s,v25.4s
+
+       b.lo    .Ltail_neon
+
+       add     x5,x5,x6,lsl#32 // pack
+       add     x7,x7,x8,lsl#32
+       ldp     x6,x8,[x1,#0]           // load input
+       add     x9,x9,x10,lsl#32
+       add     x11,x11,x12,lsl#32
+       ldp     x10,x12,[x1,#16]
+       add     x13,x13,x14,lsl#32
+       add     x15,x15,x16,lsl#32
+       ldp     x14,x16,[x1,#32]
+       add     x17,x17,x19,lsl#32
+       add     x20,x20,x21,lsl#32
+       ldp     x19,x21,[x1,#48]
+       add     x1,x1,#64
+#ifdef __ARMEB__
+       rev     x5,x5
+       rev     x7,x7
+       rev     x9,x9
+       rev     x11,x11
+       rev     x13,x13
+       rev     x15,x15
+       rev     x17,x17
+       rev     x20,x20
+#endif
+       ld1     {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64
+       eor     x5,x5,x6
+       eor     x7,x7,x8
+       eor     x9,x9,x10
+       eor     x11,x11,x12
+       eor     x13,x13,x14
+       eor     v0.16b,v0.16b,v20.16b
+       eor     x15,x15,x16
+       eor     v1.16b,v1.16b,v21.16b
+       eor     x17,x17,x19
+       eor     v2.16b,v2.16b,v22.16b
+       eor     x20,x20,x21
+       eor     v3.16b,v3.16b,v23.16b
+       ld1     {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64
+
+       stp     x5,x7,[x0,#0]           // store output
+       add     x28,x28,#4                      // increment counter
+       stp     x9,x11,[x0,#16]
+       add     v27.4s,v27.4s,v31.4s            // += 4
+       stp     x13,x15,[x0,#32]
+       add     v28.4s,v28.4s,v31.4s
+       stp     x17,x20,[x0,#48]
+       add     v29.4s,v29.4s,v31.4s
+       add     x0,x0,#64
+
+       st1     {v0.16b,v1.16b,v2.16b,v3.16b},[x0],#64
+       ld1     {v0.16b,v1.16b,v2.16b,v3.16b},[x1],#64
+
+       eor     v4.16b,v4.16b,v20.16b
+       eor     v5.16b,v5.16b,v21.16b
+       eor     v6.16b,v6.16b,v22.16b
+       eor     v7.16b,v7.16b,v23.16b
+       st1     {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64
+
+       eor     v16.16b,v16.16b,v0.16b
+       eor     v17.16b,v17.16b,v1.16b
+       eor     v18.16b,v18.16b,v2.16b
+       eor     v19.16b,v19.16b,v3.16b
+       st1     {v16.16b,v17.16b,v18.16b,v19.16b},[x0],#64
+
+       b.hi    .Loop_outer_neon
+
+       ldp     x19,x20,[x29,#16]
+       add     sp,sp,#64
+       ldp     x21,x22,[x29,#32]
+       ldp     x23,x24,[x29,#48]
+       ldp     x25,x26,[x29,#64]
+       ldp     x27,x28,[x29,#80]
+       ldp     x29,x30,[sp],#96
+       ret
+
+.Ltail_neon:
+       add     x2,x2,#256
+       cmp     x2,#64
+       b.lo    .Less_than_64
+
+       add     x5,x5,x6,lsl#32 // pack
+       add     x7,x7,x8,lsl#32
+       ldp     x6,x8,[x1,#0]           // load input
+       add     x9,x9,x10,lsl#32
+       add     x11,x11,x12,lsl#32
+       ldp     x10,x12,[x1,#16]
+       add     x13,x13,x14,lsl#32
+       add     x15,x15,x16,lsl#32
+       ldp     x14,x16,[x1,#32]
+       add     x17,x17,x19,lsl#32
+       add     x20,x20,x21,lsl#32
+       ldp     x19,x21,[x1,#48]
+       add     x1,x1,#64
+#ifdef __ARMEB__
+       rev     x5,x5
+       rev     x7,x7
+       rev     x9,x9
+       rev     x11,x11
+       rev     x13,x13
+       rev     x15,x15
+       rev     x17,x17
+       rev     x20,x20
+#endif
+       eor     x5,x5,x6
+       eor     x7,x7,x8
+       eor     x9,x9,x10
+       eor     x11,x11,x12
+       eor     x13,x13,x14
+       eor     x15,x15,x16
+       eor     x17,x17,x19
+       eor     x20,x20,x21
+
+       stp     x5,x7,[x0,#0]           // store output
+       add     x28,x28,#4                      // increment counter
+       stp     x9,x11,[x0,#16]
+       stp     x13,x15,[x0,#32]
+       stp     x17,x20,[x0,#48]
+       add     x0,x0,#64
+       b.eq    .Ldone_neon
+       sub     x2,x2,#64
+       cmp     x2,#64
+       b.lo    .Less_than_128
+
+       ld1     {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64
+       eor     v0.16b,v0.16b,v20.16b
+       eor     v1.16b,v1.16b,v21.16b
+       eor     v2.16b,v2.16b,v22.16b
+       eor     v3.16b,v3.16b,v23.16b
+       st1     {v0.16b,v1.16b,v2.16b,v3.16b},[x0],#64
+       b.eq    .Ldone_neon
+       sub     x2,x2,#64
+       cmp     x2,#64
+       b.lo    .Less_than_192
+
+       ld1     {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64
+       eor     v4.16b,v4.16b,v20.16b
+       eor     v5.16b,v5.16b,v21.16b
+       eor     v6.16b,v6.16b,v22.16b
+       eor     v7.16b,v7.16b,v23.16b
+       st1     {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64
+       b.eq    .Ldone_neon
+       sub     x2,x2,#64
+
+       st1     {v16.16b,v17.16b,v18.16b,v19.16b},[sp]
+       b       .Last_neon
+
+.Less_than_128:
+       st1     {v0.16b,v1.16b,v2.16b,v3.16b},[sp]
+       b       .Last_neon
+.Less_than_192:
+       st1     {v4.16b,v5.16b,v6.16b,v7.16b},[sp]
+       b       .Last_neon
+
+.align 4
+.Last_neon:
+       sub     x0,x0,#1
+       add     x1,x1,x2
+       add     x0,x0,x2
+       add     x4,sp,x2
+       neg     x2,x2
+
+.Loop_tail_neon:
+       ldrb    w10,[x1,x2]
+       ldrb    w11,[x4,x2]
+       add     x2,x2,#1
+       eor     w10,w10,w11
+       strb    w10,[x0,x2]
+       cbnz    x2,.Loop_tail_neon
+
+       stp     xzr,xzr,[sp,#0]
+       stp     xzr,xzr,[sp,#16]
+       stp     xzr,xzr,[sp,#32]
+       stp     xzr,xzr,[sp,#48]
+
+.Ldone_neon:
+       ldp     x19,x20,[x29,#16]
+       add     sp,sp,#64
+       ldp     x21,x22,[x29,#32]
+       ldp     x23,x24,[x29,#48]
+       ldp     x25,x26,[x29,#64]
+       ldp     x27,x28,[x29,#80]
+       ldp     x29,x30,[sp],#96
+       ret
+
+.L512_or_more_neon:
+       sub     sp,sp,#128+64
+
+       ldp     x22,x23,[x5]            // load sigma
+       ld1     {v24.4s},[x5],#16
+       ldp     x24,x25,[x3]            // load key
+       ldp     x26,x27,[x3,#16]
+       ld1     {v25.4s,v26.4s},[x3]
+       ldp     x28,x30,[x4]            // load counter
+       ld1     {v27.4s},[x4]
+       ld1     {v31.4s},[x5]
+#ifdef __ARMEB__
+       rev64   v24.4s,v24.4s
+       ror     x24,x24,#32
+       ror     x25,x25,#32
+       ror     x26,x26,#32
+       ror     x27,x27,#32
+       ror     x28,x28,#32
+       ror     x30,x30,#32
+#endif
+       add     v27.4s,v27.4s,v31.4s            // += 1
+       stp     q24,q25,[sp,#0]         // off-load key block, invariant part
+       add     v27.4s,v27.4s,v31.4s            // not typo
+       str     q26,[sp,#32]
+       add     v28.4s,v27.4s,v31.4s
+       add     v29.4s,v28.4s,v31.4s
+       add     v30.4s,v29.4s,v31.4s
+       shl     v31.4s,v31.4s,#2                        // 1 -> 4
+
+       stp     d8,d9,[sp,#128+0]               // meet ABI requirements
+       stp     d10,d11,[sp,#128+16]
+       stp     d12,d13,[sp,#128+32]
+       stp     d14,d15,[sp,#128+48]
+
+       sub     x2,x2,#512                      // not typo
+
+.Loop_outer_512_neon:
+       mov     v0.16b,v24.16b
+       mov     v4.16b,v24.16b
+       mov     v8.16b,v24.16b
+       mov     v12.16b,v24.16b
+       mov     v16.16b,v24.16b
+       mov     v20.16b,v24.16b
+       mov     v1.16b,v25.16b
+       mov     w5,w22                  // unpack key block
+       mov     v5.16b,v25.16b
+       lsr     x6,x22,#32
+       mov     v9.16b,v25.16b
+       mov     w7,w23
+       mov     v13.16b,v25.16b
+       lsr     x8,x23,#32
+       mov     v17.16b,v25.16b
+       mov     w9,w24
+       mov     v21.16b,v25.16b
+       lsr     x10,x24,#32
+       mov     v3.16b,v27.16b
+       mov     w11,w25
+       mov     v7.16b,v28.16b
+       lsr     x12,x25,#32
+       mov     v11.16b,v29.16b
+       mov     w13,w26
+       mov     v15.16b,v30.16b
+       lsr     x14,x26,#32
+       mov     v2.16b,v26.16b
+       mov     w15,w27
+       mov     v6.16b,v26.16b
+       lsr     x16,x27,#32
+       add     v19.4s,v3.4s,v31.4s                     // +4
+       mov     w17,w28
+       add     v23.4s,v7.4s,v31.4s                     // +4
+       lsr     x19,x28,#32
+       mov     v10.16b,v26.16b
+       mov     w20,w30
+       mov     v14.16b,v26.16b
+       lsr     x21,x30,#32
+       mov     v18.16b,v26.16b
+       stp     q27,q28,[sp,#48]                // off-load key block, variable 
part
+       mov     v22.16b,v26.16b
+       str     q29,[sp,#80]
+
+       mov     x4,#5
+       subs    x2,x2,#512
+.Loop_upper_neon:
+       sub     x4,x4,#1
+       add     v0.4s,v0.4s,v1.4s
+       add     w5,w5,w9
+       add     v4.4s,v4.4s,v5.4s
+       add     w6,w6,w10
+       add     v8.4s,v8.4s,v9.4s
+       add     w7,w7,w11
+       add     v12.4s,v12.4s,v13.4s
+       add     w8,w8,w12
+       add     v16.4s,v16.4s,v17.4s
+       eor     w17,w17,w5
+       add     v20.4s,v20.4s,v21.4s
+       eor     w19,w19,w6
+       eor     v3.16b,v3.16b,v0.16b
+       eor     w20,w20,w7
+       eor     v7.16b,v7.16b,v4.16b
+       eor     w21,w21,w8
+       eor     v11.16b,v11.16b,v8.16b
+       ror     w17,w17,#16
+       eor     v15.16b,v15.16b,v12.16b
+       ror     w19,w19,#16
+       eor     v19.16b,v19.16b,v16.16b
+       ror     w20,w20,#16
+       eor     v23.16b,v23.16b,v20.16b
+       ror     w21,w21,#16
+       rev32   v3.8h,v3.8h
+       add     w13,w13,w17
+       rev32   v7.8h,v7.8h
+       add     w14,w14,w19
+       rev32   v11.8h,v11.8h
+       add     w15,w15,w20
+       rev32   v15.8h,v15.8h
+       add     w16,w16,w21
+       rev32   v19.8h,v19.8h
+       eor     w9,w9,w13
+       rev32   v23.8h,v23.8h
+       eor     w10,w10,w14
+       add     v2.4s,v2.4s,v3.4s
+       eor     w11,w11,w15
+       add     v6.4s,v6.4s,v7.4s
+       eor     w12,w12,w16
+       add     v10.4s,v10.4s,v11.4s
+       ror     w9,w9,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w10,w10,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w11,w11,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w12,w12,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w9
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w10
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w11
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w12
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w17,w17,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w19,w19,w6
+       ushr    v1.4s,v24.4s,#20
+       eor     w20,w20,w7
+       ushr    v5.4s,v25.4s,#20
+       eor     w21,w21,w8
+       ushr    v9.4s,v26.4s,#20
+       ror     w17,w17,#24
+       ushr    v13.4s,v27.4s,#20
+       ror     w19,w19,#24
+       ushr    v17.4s,v28.4s,#20
+       ror     w20,w20,#24
+       ushr    v21.4s,v29.4s,#20
+       ror     w21,w21,#24
+       sli     v1.4s,v24.4s,#12
+       add     w13,w13,w17
+       sli     v5.4s,v25.4s,#12
+       add     w14,w14,w19
+       sli     v9.4s,v26.4s,#12
+       add     w15,w15,w20
+       sli     v13.4s,v27.4s,#12
+       add     w16,w16,w21
+       sli     v17.4s,v28.4s,#12
+       eor     w9,w9,w13
+       sli     v21.4s,v29.4s,#12
+       eor     w10,w10,w14
+       add     v0.4s,v0.4s,v1.4s
+       eor     w11,w11,w15
+       add     v4.4s,v4.4s,v5.4s
+       eor     w12,w12,w16
+       add     v8.4s,v8.4s,v9.4s
+       ror     w9,w9,#25
+       add     v12.4s,v12.4s,v13.4s
+       ror     w10,w10,#25
+       add     v16.4s,v16.4s,v17.4s
+       ror     w11,w11,#25
+       add     v20.4s,v20.4s,v21.4s
+       ror     w12,w12,#25
+       eor     v24.16b,v3.16b,v0.16b
+       add     w5,w5,w10
+       eor     v25.16b,v7.16b,v4.16b
+       add     w6,w6,w11
+       eor     v26.16b,v11.16b,v8.16b
+       add     w7,w7,w12
+       eor     v27.16b,v15.16b,v12.16b
+       add     w8,w8,w9
+       eor     v28.16b,v19.16b,v16.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v23.16b,v20.16b
+       eor     w17,w17,w6
+       ushr    v3.4s,v24.4s,#24
+       eor     w19,w19,w7
+       ushr    v7.4s,v25.4s,#24
+       eor     w20,w20,w8
+       ushr    v11.4s,v26.4s,#24
+       ror     w21,w21,#16
+       ushr    v15.4s,v27.4s,#24
+       ror     w17,w17,#16
+       ushr    v19.4s,v28.4s,#24
+       ror     w19,w19,#16
+       ushr    v23.4s,v29.4s,#24
+       ror     w20,w20,#16
+       sli     v3.4s,v24.4s,#8
+       add     w15,w15,w21
+       sli     v7.4s,v25.4s,#8
+       add     w16,w16,w17
+       sli     v11.4s,v26.4s,#8
+       add     w13,w13,w19
+       sli     v15.4s,v27.4s,#8
+       add     w14,w14,w20
+       sli     v19.4s,v28.4s,#8
+       eor     w10,w10,w15
+       sli     v23.4s,v29.4s,#8
+       eor     w11,w11,w16
+       add     v2.4s,v2.4s,v3.4s
+       eor     w12,w12,w13
+       add     v6.4s,v6.4s,v7.4s
+       eor     w9,w9,w14
+       add     v10.4s,v10.4s,v11.4s
+       ror     w10,w10,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w11,w11,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w12,w12,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w9,w9,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w10
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w11
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w12
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w9
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w17,w17,w6
+       ushr    v1.4s,v24.4s,#25
+       eor     w19,w19,w7
+       ushr    v5.4s,v25.4s,#25
+       eor     w20,w20,w8
+       ushr    v9.4s,v26.4s,#25
+       ror     w21,w21,#24
+       ushr    v13.4s,v27.4s,#25
+       ror     w17,w17,#24
+       ushr    v17.4s,v28.4s,#25
+       ror     w19,w19,#24
+       ushr    v21.4s,v29.4s,#25
+       ror     w20,w20,#24
+       sli     v1.4s,v24.4s,#7
+       add     w15,w15,w21
+       sli     v5.4s,v25.4s,#7
+       add     w16,w16,w17
+       sli     v9.4s,v26.4s,#7
+       add     w13,w13,w19
+       sli     v13.4s,v27.4s,#7
+       add     w14,w14,w20
+       sli     v17.4s,v28.4s,#7
+       eor     w10,w10,w15
+       sli     v21.4s,v29.4s,#7
+       eor     w11,w11,w16
+       ext     v2.16b,v2.16b,v2.16b,#8
+       eor     w12,w12,w13
+       ext     v6.16b,v6.16b,v6.16b,#8
+       eor     w9,w9,w14
+       ext     v10.16b,v10.16b,v10.16b,#8
+       ror     w10,w10,#25
+       ext     v14.16b,v14.16b,v14.16b,#8
+       ror     w11,w11,#25
+       ext     v18.16b,v18.16b,v18.16b,#8
+       ror     w12,w12,#25
+       ext     v22.16b,v22.16b,v22.16b,#8
+       ror     w9,w9,#25
+       ext     v3.16b,v3.16b,v3.16b,#12
+       ext     v7.16b,v7.16b,v7.16b,#12
+       ext     v11.16b,v11.16b,v11.16b,#12
+       ext     v15.16b,v15.16b,v15.16b,#12
+       ext     v19.16b,v19.16b,v19.16b,#12
+       ext     v23.16b,v23.16b,v23.16b,#12
+       ext     v1.16b,v1.16b,v1.16b,#4
+       ext     v5.16b,v5.16b,v5.16b,#4
+       ext     v9.16b,v9.16b,v9.16b,#4
+       ext     v13.16b,v13.16b,v13.16b,#4
+       ext     v17.16b,v17.16b,v17.16b,#4
+       ext     v21.16b,v21.16b,v21.16b,#4
+       add     v0.4s,v0.4s,v1.4s
+       add     w5,w5,w9
+       add     v4.4s,v4.4s,v5.4s
+       add     w6,w6,w10
+       add     v8.4s,v8.4s,v9.4s
+       add     w7,w7,w11
+       add     v12.4s,v12.4s,v13.4s
+       add     w8,w8,w12
+       add     v16.4s,v16.4s,v17.4s
+       eor     w17,w17,w5
+       add     v20.4s,v20.4s,v21.4s
+       eor     w19,w19,w6
+       eor     v3.16b,v3.16b,v0.16b
+       eor     w20,w20,w7
+       eor     v7.16b,v7.16b,v4.16b
+       eor     w21,w21,w8
+       eor     v11.16b,v11.16b,v8.16b
+       ror     w17,w17,#16
+       eor     v15.16b,v15.16b,v12.16b
+       ror     w19,w19,#16
+       eor     v19.16b,v19.16b,v16.16b
+       ror     w20,w20,#16
+       eor     v23.16b,v23.16b,v20.16b
+       ror     w21,w21,#16
+       rev32   v3.8h,v3.8h
+       add     w13,w13,w17
+       rev32   v7.8h,v7.8h
+       add     w14,w14,w19
+       rev32   v11.8h,v11.8h
+       add     w15,w15,w20
+       rev32   v15.8h,v15.8h
+       add     w16,w16,w21
+       rev32   v19.8h,v19.8h
+       eor     w9,w9,w13
+       rev32   v23.8h,v23.8h
+       eor     w10,w10,w14
+       add     v2.4s,v2.4s,v3.4s
+       eor     w11,w11,w15
+       add     v6.4s,v6.4s,v7.4s
+       eor     w12,w12,w16
+       add     v10.4s,v10.4s,v11.4s
+       ror     w9,w9,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w10,w10,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w11,w11,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w12,w12,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w9
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w10
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w11
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w12
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w17,w17,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w19,w19,w6
+       ushr    v1.4s,v24.4s,#20
+       eor     w20,w20,w7
+       ushr    v5.4s,v25.4s,#20
+       eor     w21,w21,w8
+       ushr    v9.4s,v26.4s,#20
+       ror     w17,w17,#24
+       ushr    v13.4s,v27.4s,#20
+       ror     w19,w19,#24
+       ushr    v17.4s,v28.4s,#20
+       ror     w20,w20,#24
+       ushr    v21.4s,v29.4s,#20
+       ror     w21,w21,#24
+       sli     v1.4s,v24.4s,#12
+       add     w13,w13,w17
+       sli     v5.4s,v25.4s,#12
+       add     w14,w14,w19
+       sli     v9.4s,v26.4s,#12
+       add     w15,w15,w20
+       sli     v13.4s,v27.4s,#12
+       add     w16,w16,w21
+       sli     v17.4s,v28.4s,#12
+       eor     w9,w9,w13
+       sli     v21.4s,v29.4s,#12
+       eor     w10,w10,w14
+       add     v0.4s,v0.4s,v1.4s
+       eor     w11,w11,w15
+       add     v4.4s,v4.4s,v5.4s
+       eor     w12,w12,w16
+       add     v8.4s,v8.4s,v9.4s
+       ror     w9,w9,#25
+       add     v12.4s,v12.4s,v13.4s
+       ror     w10,w10,#25
+       add     v16.4s,v16.4s,v17.4s
+       ror     w11,w11,#25
+       add     v20.4s,v20.4s,v21.4s
+       ror     w12,w12,#25
+       eor     v24.16b,v3.16b,v0.16b
+       add     w5,w5,w10
+       eor     v25.16b,v7.16b,v4.16b
+       add     w6,w6,w11
+       eor     v26.16b,v11.16b,v8.16b
+       add     w7,w7,w12
+       eor     v27.16b,v15.16b,v12.16b
+       add     w8,w8,w9
+       eor     v28.16b,v19.16b,v16.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v23.16b,v20.16b
+       eor     w17,w17,w6
+       ushr    v3.4s,v24.4s,#24
+       eor     w19,w19,w7
+       ushr    v7.4s,v25.4s,#24
+       eor     w20,w20,w8
+       ushr    v11.4s,v26.4s,#24
+       ror     w21,w21,#16
+       ushr    v15.4s,v27.4s,#24
+       ror     w17,w17,#16
+       ushr    v19.4s,v28.4s,#24
+       ror     w19,w19,#16
+       ushr    v23.4s,v29.4s,#24
+       ror     w20,w20,#16
+       sli     v3.4s,v24.4s,#8
+       add     w15,w15,w21
+       sli     v7.4s,v25.4s,#8
+       add     w16,w16,w17
+       sli     v11.4s,v26.4s,#8
+       add     w13,w13,w19
+       sli     v15.4s,v27.4s,#8
+       add     w14,w14,w20
+       sli     v19.4s,v28.4s,#8
+       eor     w10,w10,w15
+       sli     v23.4s,v29.4s,#8
+       eor     w11,w11,w16
+       add     v2.4s,v2.4s,v3.4s
+       eor     w12,w12,w13
+       add     v6.4s,v6.4s,v7.4s
+       eor     w9,w9,w14
+       add     v10.4s,v10.4s,v11.4s
+       ror     w10,w10,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w11,w11,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w12,w12,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w9,w9,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w10
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w11
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w12
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w9
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w17,w17,w6
+       ushr    v1.4s,v24.4s,#25
+       eor     w19,w19,w7
+       ushr    v5.4s,v25.4s,#25
+       eor     w20,w20,w8
+       ushr    v9.4s,v26.4s,#25
+       ror     w21,w21,#24
+       ushr    v13.4s,v27.4s,#25
+       ror     w17,w17,#24
+       ushr    v17.4s,v28.4s,#25
+       ror     w19,w19,#24
+       ushr    v21.4s,v29.4s,#25
+       ror     w20,w20,#24
+       sli     v1.4s,v24.4s,#7
+       add     w15,w15,w21
+       sli     v5.4s,v25.4s,#7
+       add     w16,w16,w17
+       sli     v9.4s,v26.4s,#7
+       add     w13,w13,w19
+       sli     v13.4s,v27.4s,#7
+       add     w14,w14,w20
+       sli     v17.4s,v28.4s,#7
+       eor     w10,w10,w15
+       sli     v21.4s,v29.4s,#7
+       eor     w11,w11,w16
+       ext     v2.16b,v2.16b,v2.16b,#8
+       eor     w12,w12,w13
+       ext     v6.16b,v6.16b,v6.16b,#8
+       eor     w9,w9,w14
+       ext     v10.16b,v10.16b,v10.16b,#8
+       ror     w10,w10,#25
+       ext     v14.16b,v14.16b,v14.16b,#8
+       ror     w11,w11,#25
+       ext     v18.16b,v18.16b,v18.16b,#8
+       ror     w12,w12,#25
+       ext     v22.16b,v22.16b,v22.16b,#8
+       ror     w9,w9,#25
+       ext     v3.16b,v3.16b,v3.16b,#4
+       ext     v7.16b,v7.16b,v7.16b,#4
+       ext     v11.16b,v11.16b,v11.16b,#4
+       ext     v15.16b,v15.16b,v15.16b,#4
+       ext     v19.16b,v19.16b,v19.16b,#4
+       ext     v23.16b,v23.16b,v23.16b,#4
+       ext     v1.16b,v1.16b,v1.16b,#12
+       ext     v5.16b,v5.16b,v5.16b,#12
+       ext     v9.16b,v9.16b,v9.16b,#12
+       ext     v13.16b,v13.16b,v13.16b,#12
+       ext     v17.16b,v17.16b,v17.16b,#12
+       ext     v21.16b,v21.16b,v21.16b,#12
+       cbnz    x4,.Loop_upper_neon
+
+       add     w5,w5,w22               // accumulate key block
+       add     x6,x6,x22,lsr#32
+       add     w7,w7,w23
+       add     x8,x8,x23,lsr#32
+       add     w9,w9,w24
+       add     x10,x10,x24,lsr#32
+       add     w11,w11,w25
+       add     x12,x12,x25,lsr#32
+       add     w13,w13,w26
+       add     x14,x14,x26,lsr#32
+       add     w15,w15,w27
+       add     x16,x16,x27,lsr#32
+       add     w17,w17,w28
+       add     x19,x19,x28,lsr#32
+       add     w20,w20,w30
+       add     x21,x21,x30,lsr#32
+
+       add     x5,x5,x6,lsl#32 // pack
+       add     x7,x7,x8,lsl#32
+       ldp     x6,x8,[x1,#0]           // load input
+       add     x9,x9,x10,lsl#32
+       add     x11,x11,x12,lsl#32
+       ldp     x10,x12,[x1,#16]
+       add     x13,x13,x14,lsl#32
+       add     x15,x15,x16,lsl#32
+       ldp     x14,x16,[x1,#32]
+       add     x17,x17,x19,lsl#32
+       add     x20,x20,x21,lsl#32
+       ldp     x19,x21,[x1,#48]
+       add     x1,x1,#64
+#ifdef __ARMEB__
+       rev     x5,x5
+       rev     x7,x7
+       rev     x9,x9
+       rev     x11,x11
+       rev     x13,x13
+       rev     x15,x15
+       rev     x17,x17
+       rev     x20,x20
+#endif
+       eor     x5,x5,x6
+       eor     x7,x7,x8
+       eor     x9,x9,x10
+       eor     x11,x11,x12
+       eor     x13,x13,x14
+       eor     x15,x15,x16
+       eor     x17,x17,x19
+       eor     x20,x20,x21
+
+       stp     x5,x7,[x0,#0]           // store output
+       add     x28,x28,#1                      // increment counter
+       mov     w5,w22                  // unpack key block
+       lsr     x6,x22,#32
+       stp     x9,x11,[x0,#16]
+       mov     w7,w23
+       lsr     x8,x23,#32
+       stp     x13,x15,[x0,#32]
+       mov     w9,w24
+       lsr     x10,x24,#32
+       stp     x17,x20,[x0,#48]
+       add     x0,x0,#64
+       mov     w11,w25
+       lsr     x12,x25,#32
+       mov     w13,w26
+       lsr     x14,x26,#32
+       mov     w15,w27
+       lsr     x16,x27,#32
+       mov     w17,w28
+       lsr     x19,x28,#32
+       mov     w20,w30
+       lsr     x21,x30,#32
+
+       mov     x4,#5
+.Loop_lower_neon:
+       sub     x4,x4,#1
+       add     v0.4s,v0.4s,v1.4s
+       add     w5,w5,w9
+       add     v4.4s,v4.4s,v5.4s
+       add     w6,w6,w10
+       add     v8.4s,v8.4s,v9.4s
+       add     w7,w7,w11
+       add     v12.4s,v12.4s,v13.4s
+       add     w8,w8,w12
+       add     v16.4s,v16.4s,v17.4s
+       eor     w17,w17,w5
+       add     v20.4s,v20.4s,v21.4s
+       eor     w19,w19,w6
+       eor     v3.16b,v3.16b,v0.16b
+       eor     w20,w20,w7
+       eor     v7.16b,v7.16b,v4.16b
+       eor     w21,w21,w8
+       eor     v11.16b,v11.16b,v8.16b
+       ror     w17,w17,#16
+       eor     v15.16b,v15.16b,v12.16b
+       ror     w19,w19,#16
+       eor     v19.16b,v19.16b,v16.16b
+       ror     w20,w20,#16
+       eor     v23.16b,v23.16b,v20.16b
+       ror     w21,w21,#16
+       rev32   v3.8h,v3.8h
+       add     w13,w13,w17
+       rev32   v7.8h,v7.8h
+       add     w14,w14,w19
+       rev32   v11.8h,v11.8h
+       add     w15,w15,w20
+       rev32   v15.8h,v15.8h
+       add     w16,w16,w21
+       rev32   v19.8h,v19.8h
+       eor     w9,w9,w13
+       rev32   v23.8h,v23.8h
+       eor     w10,w10,w14
+       add     v2.4s,v2.4s,v3.4s
+       eor     w11,w11,w15
+       add     v6.4s,v6.4s,v7.4s
+       eor     w12,w12,w16
+       add     v10.4s,v10.4s,v11.4s
+       ror     w9,w9,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w10,w10,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w11,w11,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w12,w12,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w9
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w10
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w11
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w12
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w17,w17,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w19,w19,w6
+       ushr    v1.4s,v24.4s,#20
+       eor     w20,w20,w7
+       ushr    v5.4s,v25.4s,#20
+       eor     w21,w21,w8
+       ushr    v9.4s,v26.4s,#20
+       ror     w17,w17,#24
+       ushr    v13.4s,v27.4s,#20
+       ror     w19,w19,#24
+       ushr    v17.4s,v28.4s,#20
+       ror     w20,w20,#24
+       ushr    v21.4s,v29.4s,#20
+       ror     w21,w21,#24
+       sli     v1.4s,v24.4s,#12
+       add     w13,w13,w17
+       sli     v5.4s,v25.4s,#12
+       add     w14,w14,w19
+       sli     v9.4s,v26.4s,#12
+       add     w15,w15,w20
+       sli     v13.4s,v27.4s,#12
+       add     w16,w16,w21
+       sli     v17.4s,v28.4s,#12
+       eor     w9,w9,w13
+       sli     v21.4s,v29.4s,#12
+       eor     w10,w10,w14
+       add     v0.4s,v0.4s,v1.4s
+       eor     w11,w11,w15
+       add     v4.4s,v4.4s,v5.4s
+       eor     w12,w12,w16
+       add     v8.4s,v8.4s,v9.4s
+       ror     w9,w9,#25
+       add     v12.4s,v12.4s,v13.4s
+       ror     w10,w10,#25
+       add     v16.4s,v16.4s,v17.4s
+       ror     w11,w11,#25
+       add     v20.4s,v20.4s,v21.4s
+       ror     w12,w12,#25
+       eor     v24.16b,v3.16b,v0.16b
+       add     w5,w5,w10
+       eor     v25.16b,v7.16b,v4.16b
+       add     w6,w6,w11
+       eor     v26.16b,v11.16b,v8.16b
+       add     w7,w7,w12
+       eor     v27.16b,v15.16b,v12.16b
+       add     w8,w8,w9
+       eor     v28.16b,v19.16b,v16.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v23.16b,v20.16b
+       eor     w17,w17,w6
+       ushr    v3.4s,v24.4s,#24
+       eor     w19,w19,w7
+       ushr    v7.4s,v25.4s,#24
+       eor     w20,w20,w8
+       ushr    v11.4s,v26.4s,#24
+       ror     w21,w21,#16
+       ushr    v15.4s,v27.4s,#24
+       ror     w17,w17,#16
+       ushr    v19.4s,v28.4s,#24
+       ror     w19,w19,#16
+       ushr    v23.4s,v29.4s,#24
+       ror     w20,w20,#16
+       sli     v3.4s,v24.4s,#8
+       add     w15,w15,w21
+       sli     v7.4s,v25.4s,#8
+       add     w16,w16,w17
+       sli     v11.4s,v26.4s,#8
+       add     w13,w13,w19
+       sli     v15.4s,v27.4s,#8
+       add     w14,w14,w20
+       sli     v19.4s,v28.4s,#8
+       eor     w10,w10,w15
+       sli     v23.4s,v29.4s,#8
+       eor     w11,w11,w16
+       add     v2.4s,v2.4s,v3.4s
+       eor     w12,w12,w13
+       add     v6.4s,v6.4s,v7.4s
+       eor     w9,w9,w14
+       add     v10.4s,v10.4s,v11.4s
+       ror     w10,w10,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w11,w11,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w12,w12,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w9,w9,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w10
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w11
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w12
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w9
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w17,w17,w6
+       ushr    v1.4s,v24.4s,#25
+       eor     w19,w19,w7
+       ushr    v5.4s,v25.4s,#25
+       eor     w20,w20,w8
+       ushr    v9.4s,v26.4s,#25
+       ror     w21,w21,#24
+       ushr    v13.4s,v27.4s,#25
+       ror     w17,w17,#24
+       ushr    v17.4s,v28.4s,#25
+       ror     w19,w19,#24
+       ushr    v21.4s,v29.4s,#25
+       ror     w20,w20,#24
+       sli     v1.4s,v24.4s,#7
+       add     w15,w15,w21
+       sli     v5.4s,v25.4s,#7
+       add     w16,w16,w17
+       sli     v9.4s,v26.4s,#7
+       add     w13,w13,w19
+       sli     v13.4s,v27.4s,#7
+       add     w14,w14,w20
+       sli     v17.4s,v28.4s,#7
+       eor     w10,w10,w15
+       sli     v21.4s,v29.4s,#7
+       eor     w11,w11,w16
+       ext     v2.16b,v2.16b,v2.16b,#8
+       eor     w12,w12,w13
+       ext     v6.16b,v6.16b,v6.16b,#8
+       eor     w9,w9,w14
+       ext     v10.16b,v10.16b,v10.16b,#8
+       ror     w10,w10,#25
+       ext     v14.16b,v14.16b,v14.16b,#8
+       ror     w11,w11,#25
+       ext     v18.16b,v18.16b,v18.16b,#8
+       ror     w12,w12,#25
+       ext     v22.16b,v22.16b,v22.16b,#8
+       ror     w9,w9,#25
+       ext     v3.16b,v3.16b,v3.16b,#12
+       ext     v7.16b,v7.16b,v7.16b,#12
+       ext     v11.16b,v11.16b,v11.16b,#12
+       ext     v15.16b,v15.16b,v15.16b,#12
+       ext     v19.16b,v19.16b,v19.16b,#12
+       ext     v23.16b,v23.16b,v23.16b,#12
+       ext     v1.16b,v1.16b,v1.16b,#4
+       ext     v5.16b,v5.16b,v5.16b,#4
+       ext     v9.16b,v9.16b,v9.16b,#4
+       ext     v13.16b,v13.16b,v13.16b,#4
+       ext     v17.16b,v17.16b,v17.16b,#4
+       ext     v21.16b,v21.16b,v21.16b,#4
+       add     v0.4s,v0.4s,v1.4s
+       add     w5,w5,w9
+       add     v4.4s,v4.4s,v5.4s
+       add     w6,w6,w10
+       add     v8.4s,v8.4s,v9.4s
+       add     w7,w7,w11
+       add     v12.4s,v12.4s,v13.4s
+       add     w8,w8,w12
+       add     v16.4s,v16.4s,v17.4s
+       eor     w17,w17,w5
+       add     v20.4s,v20.4s,v21.4s
+       eor     w19,w19,w6
+       eor     v3.16b,v3.16b,v0.16b
+       eor     w20,w20,w7
+       eor     v7.16b,v7.16b,v4.16b
+       eor     w21,w21,w8
+       eor     v11.16b,v11.16b,v8.16b
+       ror     w17,w17,#16
+       eor     v15.16b,v15.16b,v12.16b
+       ror     w19,w19,#16
+       eor     v19.16b,v19.16b,v16.16b
+       ror     w20,w20,#16
+       eor     v23.16b,v23.16b,v20.16b
+       ror     w21,w21,#16
+       rev32   v3.8h,v3.8h
+       add     w13,w13,w17
+       rev32   v7.8h,v7.8h
+       add     w14,w14,w19
+       rev32   v11.8h,v11.8h
+       add     w15,w15,w20
+       rev32   v15.8h,v15.8h
+       add     w16,w16,w21
+       rev32   v19.8h,v19.8h
+       eor     w9,w9,w13
+       rev32   v23.8h,v23.8h
+       eor     w10,w10,w14
+       add     v2.4s,v2.4s,v3.4s
+       eor     w11,w11,w15
+       add     v6.4s,v6.4s,v7.4s
+       eor     w12,w12,w16
+       add     v10.4s,v10.4s,v11.4s
+       ror     w9,w9,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w10,w10,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w11,w11,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w12,w12,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w9
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w10
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w11
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w12
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w17,w17,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w19,w19,w6
+       ushr    v1.4s,v24.4s,#20
+       eor     w20,w20,w7
+       ushr    v5.4s,v25.4s,#20
+       eor     w21,w21,w8
+       ushr    v9.4s,v26.4s,#20
+       ror     w17,w17,#24
+       ushr    v13.4s,v27.4s,#20
+       ror     w19,w19,#24
+       ushr    v17.4s,v28.4s,#20
+       ror     w20,w20,#24
+       ushr    v21.4s,v29.4s,#20
+       ror     w21,w21,#24
+       sli     v1.4s,v24.4s,#12
+       add     w13,w13,w17
+       sli     v5.4s,v25.4s,#12
+       add     w14,w14,w19
+       sli     v9.4s,v26.4s,#12
+       add     w15,w15,w20
+       sli     v13.4s,v27.4s,#12
+       add     w16,w16,w21
+       sli     v17.4s,v28.4s,#12
+       eor     w9,w9,w13
+       sli     v21.4s,v29.4s,#12
+       eor     w10,w10,w14
+       add     v0.4s,v0.4s,v1.4s
+       eor     w11,w11,w15
+       add     v4.4s,v4.4s,v5.4s
+       eor     w12,w12,w16
+       add     v8.4s,v8.4s,v9.4s
+       ror     w9,w9,#25
+       add     v12.4s,v12.4s,v13.4s
+       ror     w10,w10,#25
+       add     v16.4s,v16.4s,v17.4s
+       ror     w11,w11,#25
+       add     v20.4s,v20.4s,v21.4s
+       ror     w12,w12,#25
+       eor     v24.16b,v3.16b,v0.16b
+       add     w5,w5,w10
+       eor     v25.16b,v7.16b,v4.16b
+       add     w6,w6,w11
+       eor     v26.16b,v11.16b,v8.16b
+       add     w7,w7,w12
+       eor     v27.16b,v15.16b,v12.16b
+       add     w8,w8,w9
+       eor     v28.16b,v19.16b,v16.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v23.16b,v20.16b
+       eor     w17,w17,w6
+       ushr    v3.4s,v24.4s,#24
+       eor     w19,w19,w7
+       ushr    v7.4s,v25.4s,#24
+       eor     w20,w20,w8
+       ushr    v11.4s,v26.4s,#24
+       ror     w21,w21,#16
+       ushr    v15.4s,v27.4s,#24
+       ror     w17,w17,#16
+       ushr    v19.4s,v28.4s,#24
+       ror     w19,w19,#16
+       ushr    v23.4s,v29.4s,#24
+       ror     w20,w20,#16
+       sli     v3.4s,v24.4s,#8
+       add     w15,w15,w21
+       sli     v7.4s,v25.4s,#8
+       add     w16,w16,w17
+       sli     v11.4s,v26.4s,#8
+       add     w13,w13,w19
+       sli     v15.4s,v27.4s,#8
+       add     w14,w14,w20
+       sli     v19.4s,v28.4s,#8
+       eor     w10,w10,w15
+       sli     v23.4s,v29.4s,#8
+       eor     w11,w11,w16
+       add     v2.4s,v2.4s,v3.4s
+       eor     w12,w12,w13
+       add     v6.4s,v6.4s,v7.4s
+       eor     w9,w9,w14
+       add     v10.4s,v10.4s,v11.4s
+       ror     w10,w10,#20
+       add     v14.4s,v14.4s,v15.4s
+       ror     w11,w11,#20
+       add     v18.4s,v18.4s,v19.4s
+       ror     w12,w12,#20
+       add     v22.4s,v22.4s,v23.4s
+       ror     w9,w9,#20
+       eor     v24.16b,v1.16b,v2.16b
+       add     w5,w5,w10
+       eor     v25.16b,v5.16b,v6.16b
+       add     w6,w6,w11
+       eor     v26.16b,v9.16b,v10.16b
+       add     w7,w7,w12
+       eor     v27.16b,v13.16b,v14.16b
+       add     w8,w8,w9
+       eor     v28.16b,v17.16b,v18.16b
+       eor     w21,w21,w5
+       eor     v29.16b,v21.16b,v22.16b
+       eor     w17,w17,w6
+       ushr    v1.4s,v24.4s,#25
+       eor     w19,w19,w7
+       ushr    v5.4s,v25.4s,#25
+       eor     w20,w20,w8
+       ushr    v9.4s,v26.4s,#25
+       ror     w21,w21,#24
+       ushr    v13.4s,v27.4s,#25
+       ror     w17,w17,#24
+       ushr    v17.4s,v28.4s,#25
+       ror     w19,w19,#24
+       ushr    v21.4s,v29.4s,#25
+       ror     w20,w20,#24
+       sli     v1.4s,v24.4s,#7
+       add     w15,w15,w21
+       sli     v5.4s,v25.4s,#7
+       add     w16,w16,w17
+       sli     v9.4s,v26.4s,#7
+       add     w13,w13,w19
+       sli     v13.4s,v27.4s,#7
+       add     w14,w14,w20
+       sli     v17.4s,v28.4s,#7
+       eor     w10,w10,w15
+       sli     v21.4s,v29.4s,#7
+       eor     w11,w11,w16
+       ext     v2.16b,v2.16b,v2.16b,#8
+       eor     w12,w12,w13
+       ext     v6.16b,v6.16b,v6.16b,#8
+       eor     w9,w9,w14
+       ext     v10.16b,v10.16b,v10.16b,#8
+       ror     w10,w10,#25
+       ext     v14.16b,v14.16b,v14.16b,#8
+       ror     w11,w11,#25
+       ext     v18.16b,v18.16b,v18.16b,#8
+       ror     w12,w12,#25
+       ext     v22.16b,v22.16b,v22.16b,#8
+       ror     w9,w9,#25
+       ext     v3.16b,v3.16b,v3.16b,#4
+       ext     v7.16b,v7.16b,v7.16b,#4
+       ext     v11.16b,v11.16b,v11.16b,#4
+       ext     v15.16b,v15.16b,v15.16b,#4
+       ext     v19.16b,v19.16b,v19.16b,#4
+       ext     v23.16b,v23.16b,v23.16b,#4
+       ext     v1.16b,v1.16b,v1.16b,#12
+       ext     v5.16b,v5.16b,v5.16b,#12
+       ext     v9.16b,v9.16b,v9.16b,#12
+       ext     v13.16b,v13.16b,v13.16b,#12
+       ext     v17.16b,v17.16b,v17.16b,#12
+       ext     v21.16b,v21.16b,v21.16b,#12
+       cbnz    x4,.Loop_lower_neon
+
+       add     w5,w5,w22               // accumulate key block
+       ldp     q24,q25,[sp,#0]
+       add     x6,x6,x22,lsr#32
+       ldp     q26,q27,[sp,#32]
+       add     w7,w7,w23
+       ldp     q28,q29,[sp,#64]
+       add     x8,x8,x23,lsr#32
+       add     v0.4s,v0.4s,v24.4s
+       add     w9,w9,w24
+       add     v4.4s,v4.4s,v24.4s
+       add     x10,x10,x24,lsr#32
+       add     v8.4s,v8.4s,v24.4s
+       add     w11,w11,w25
+       add     v12.4s,v12.4s,v24.4s
+       add     x12,x12,x25,lsr#32
+       add     v16.4s,v16.4s,v24.4s
+       add     w13,w13,w26
+       add     v20.4s,v20.4s,v24.4s
+       add     x14,x14,x26,lsr#32
+       add     v2.4s,v2.4s,v26.4s
+       add     w15,w15,w27
+       add     v6.4s,v6.4s,v26.4s
+       add     x16,x16,x27,lsr#32
+       add     v10.4s,v10.4s,v26.4s
+       add     w17,w17,w28
+       add     v14.4s,v14.4s,v26.4s
+       add     x19,x19,x28,lsr#32
+       add     v18.4s,v18.4s,v26.4s
+       add     w20,w20,w30
+       add     v22.4s,v22.4s,v26.4s
+       add     x21,x21,x30,lsr#32
+       add     v19.4s,v19.4s,v31.4s                    // +4
+       add     x5,x5,x6,lsl#32 // pack
+       add     v23.4s,v23.4s,v31.4s                    // +4
+       add     x7,x7,x8,lsl#32
+       add     v3.4s,v3.4s,v27.4s
+       ldp     x6,x8,[x1,#0]           // load input
+       add     v7.4s,v7.4s,v28.4s
+       add     x9,x9,x10,lsl#32
+       add     v11.4s,v11.4s,v29.4s
+       add     x11,x11,x12,lsl#32
+       add     v15.4s,v15.4s,v30.4s
+       ldp     x10,x12,[x1,#16]
+       add     v19.4s,v19.4s,v27.4s
+       add     x13,x13,x14,lsl#32
+       add     v23.4s,v23.4s,v28.4s
+       add     x15,x15,x16,lsl#32
+       add     v1.4s,v1.4s,v25.4s
+       ldp     x14,x16,[x1,#32]
+       add     v5.4s,v5.4s,v25.4s
+       add     x17,x17,x19,lsl#32
+       add     v9.4s,v9.4s,v25.4s
+       add     x20,x20,x21,lsl#32
+       add     v13.4s,v13.4s,v25.4s
+       ldp     x19,x21,[x1,#48]
+       add     v17.4s,v17.4s,v25.4s
+       add     x1,x1,#64
+       add     v21.4s,v21.4s,v25.4s
+
+#ifdef __ARMEB__
+       rev     x5,x5
+       rev     x7,x7
+       rev     x9,x9
+       rev     x11,x11
+       rev     x13,x13
+       rev     x15,x15
+       rev     x17,x17
+       rev     x20,x20
+#endif
+       ld1     {v24.16b,v25.16b,v26.16b,v27.16b},[x1],#64
+       eor     x5,x5,x6
+       eor     x7,x7,x8
+       eor     x9,x9,x10
+       eor     x11,x11,x12
+       eor     x13,x13,x14
+       eor     v0.16b,v0.16b,v24.16b
+       eor     x15,x15,x16
+       eor     v1.16b,v1.16b,v25.16b
+       eor     x17,x17,x19
+       eor     v2.16b,v2.16b,v26.16b
+       eor     x20,x20,x21
+       eor     v3.16b,v3.16b,v27.16b
+       ld1     {v24.16b,v25.16b,v26.16b,v27.16b},[x1],#64
+
+       stp     x5,x7,[x0,#0]           // store output
+       add     x28,x28,#7                      // increment counter
+       stp     x9,x11,[x0,#16]
+       stp     x13,x15,[x0,#32]
+       stp     x17,x20,[x0,#48]
+       add     x0,x0,#64
+       st1     {v0.16b,v1.16b,v2.16b,v3.16b},[x0],#64
+
+       ld1     {v0.16b,v1.16b,v2.16b,v3.16b},[x1],#64
+       eor     v4.16b,v4.16b,v24.16b
+       eor     v5.16b,v5.16b,v25.16b
+       eor     v6.16b,v6.16b,v26.16b
+       eor     v7.16b,v7.16b,v27.16b
+       st1     {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64
+
+       ld1     {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64
+       eor     v8.16b,v8.16b,v0.16b
+       ldp     q24,q25,[sp,#0]
+       eor     v9.16b,v9.16b,v1.16b
+       ldp     q26,q27,[sp,#32]
+       eor     v10.16b,v10.16b,v2.16b
+       eor     v11.16b,v11.16b,v3.16b
+       st1     {v8.16b,v9.16b,v10.16b,v11.16b},[x0],#64
+
+       ld1     {v8.16b,v9.16b,v10.16b,v11.16b},[x1],#64
+       eor     v12.16b,v12.16b,v4.16b
+       eor     v13.16b,v13.16b,v5.16b
+       eor     v14.16b,v14.16b,v6.16b
+       eor     v15.16b,v15.16b,v7.16b
+       st1     {v12.16b,v13.16b,v14.16b,v15.16b},[x0],#64
+
+       ld1     {v12.16b,v13.16b,v14.16b,v15.16b},[x1],#64
+       eor     v16.16b,v16.16b,v8.16b
+       eor     v17.16b,v17.16b,v9.16b
+       eor     v18.16b,v18.16b,v10.16b
+       eor     v19.16b,v19.16b,v11.16b
+       st1     {v16.16b,v17.16b,v18.16b,v19.16b},[x0],#64
+
+       shl     v0.4s,v31.4s,#1                 // 4 -> 8
+       eor     v20.16b,v20.16b,v12.16b
+       eor     v21.16b,v21.16b,v13.16b
+       eor     v22.16b,v22.16b,v14.16b
+       eor     v23.16b,v23.16b,v15.16b
+       st1     {v20.16b,v21.16b,v22.16b,v23.16b},[x0],#64
+
+       add     v27.4s,v27.4s,v0.4s                     // += 8
+       add     v28.4s,v28.4s,v0.4s
+       add     v29.4s,v29.4s,v0.4s
+       add     v30.4s,v30.4s,v0.4s
+
+       b.hs    .Loop_outer_512_neon
+
+       adds    x2,x2,#512
+       ushr    v0.4s,v31.4s,#2                 // 4 -> 1
+
+       ldp     d8,d9,[sp,#128+0]               // meet ABI requirements
+       ldp     d10,d11,[sp,#128+16]
+       ldp     d12,d13,[sp,#128+32]
+       ldp     d14,d15,[sp,#128+48]
+
+       stp     q24,q31,[sp,#0]         // wipe off-load area
+       stp     q24,q31,[sp,#32]
+       stp     q24,q31,[sp,#64]
+
+       b.eq    .Ldone_512_neon
+
+       cmp     x2,#192
+       sub     v27.4s,v27.4s,v0.4s                     // -= 1
+       sub     v28.4s,v28.4s,v0.4s
+       sub     v29.4s,v29.4s,v0.4s
+       add     sp,sp,#128
+       b.hs    .Loop_outer_neon
+
+       eor     v25.16b,v25.16b,v25.16b
+       eor     v26.16b,v26.16b,v26.16b
+       eor     v27.16b,v27.16b,v27.16b
+       eor     v28.16b,v28.16b,v28.16b
+       eor     v29.16b,v29.16b,v29.16b
+       eor     v30.16b,v30.16b,v30.16b
+       b       .Loop_outer
+
+.Ldone_512_neon:
+       ldp     x19,x20,[x29,#16]
+       add     sp,sp,#128+64
+       ldp     x21,x22,[x29,#32]
+       ldp     x23,x24,[x29,#48]
+       ldp     x25,x26,[x29,#64]
+       ldp     x27,x28,[x29,#80]
+       ldp     x29,x30,[sp],#96
+.Labort_neon:
+       ret
+ENDPROC(chacha20_neon)
-- 
2.18.0

Reply via email to