LXR linux/arch/arm/crypto/sha1-armv7-neon.S

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/* sha1-armv7-neon.S - ARM/NEON accelerated SHA-1 transform function
   3 *
   4 * Copyright © 2013-2014 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   5 */
   6
   7#include <linux/linkage.h>
   8#include <asm/assembler.h>
   9
  10.syntax unified
  11.fpu neon
  12
  13.text
  14
  15
  16/* Context structure */
  17
  18#define state_h0 0
  19#define state_h1 4
  20#define state_h2 8
  21#define state_h3 12
  22#define state_h4 16
  23
  24
  25/* Constants */
  26
  27#define K1  0x5A827999
  28#define K2  0x6ED9EBA1
  29#define K3  0x8F1BBCDC
  30#define K4  0xCA62C1D6
  31.align 4
  32.LK_VEC:
  33.LK1:   .long K1, K1, K1, K1
  34.LK2:   .long K2, K2, K2, K2
  35.LK3:   .long K3, K3, K3, K3
  36.LK4:   .long K4, K4, K4, K4
  37
  38
  39/* Register macros */
  40
  41#define RSTATE r0
  42#define RDATA r1
  43#define RNBLKS r2
  44#define ROLDSTACK r3
  45#define RWK lr
  46
  47#define _a r4
  48#define _b r5
  49#define _c r6
  50#define _d r7
  51#define _e r8
  52
  53#define RT0 r9
  54#define RT1 r10
  55#define RT2 r11
  56#define RT3 r12
  57
  58#define W0 q0
  59#define W1 q7
  60#define W2 q2
  61#define W3 q3
  62#define W4 q4
  63#define W5 q6
  64#define W6 q5
  65#define W7 q1
  66
  67#define tmp0 q8
  68#define tmp1 q9
  69#define tmp2 q10
  70#define tmp3 q11
  71
  72#define qK1 q12
  73#define qK2 q13
  74#define qK3 q14
  75#define qK4 q15
  76
  77#ifdef CONFIG_CPU_BIG_ENDIAN
  78#define ARM_LE(code...)
  79#else
  80#define ARM_LE(code...)         code
  81#endif
  82
  83/* Round function macros. */
  84
  85#define WK_offs(i) (((i) & 15) * 4)
  86
  87#define _R_F1(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
  88              W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
  89        ldr RT3, [sp, WK_offs(i)]; \
  90                pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
  91        bic RT0, d, b; \
  92        add e, e, a, ror #(32 - 5); \
  93        and RT1, c, b; \
  94                pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
  95        add RT0, RT0, RT3; \
  96        add e, e, RT1; \
  97        ror b, #(32 - 30); \
  98                pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
  99        add e, e, RT0;
 100
 101#define _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
 102              W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 103        ldr RT3, [sp, WK_offs(i)]; \
 104                pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
 105        eor RT0, d, b; \
 106        add e, e, a, ror #(32 - 5); \
 107        eor RT0, RT0, c; \
 108                pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
 109        add e, e, RT3; \
 110        ror b, #(32 - 30); \
 111                pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
 112        add e, e, RT0; \
 113
 114#define _R_F3(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
 115              W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 116        ldr RT3, [sp, WK_offs(i)]; \
 117                pre1(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
 118        eor RT0, b, c; \
 119        and RT1, b, c; \
 120        add e, e, a, ror #(32 - 5); \
 121                pre2(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
 122        and RT0, RT0, d; \
 123        add RT1, RT1, RT3; \
 124        add e, e, RT0; \
 125        ror b, #(32 - 30); \
 126                pre3(i16,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28); \
 127        add e, e, RT1;
 128
 129#define _R_F4(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
 130              W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 131        _R_F2(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
 132              W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28)
 133
 134#define _R(a,b,c,d,e,f,i,pre1,pre2,pre3,i16,\
 135           W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 136        _R_##f(a,b,c,d,e,i,pre1,pre2,pre3,i16,\
 137               W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28)
 138
 139#define R(a,b,c,d,e,f,i) \
 140        _R_##f(a,b,c,d,e,i,dummy,dummy,dummy,i16,\
 141               W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28)
 142
 143#define dummy(...)
 144
 145
 146/* Input expansion macros. */
 147
 148/********* Precalc macros for rounds 0-15 *************************************/
 149
 150#define W_PRECALC_00_15() \
 151        add       RWK, sp, #(WK_offs(0));                       \
 152        \
 153        vld1.32   {W0, W7}, [RDATA]!;                           \
 154 ARM_LE(vrev32.8  W0, W0;       )       /* big => little */     \
 155        vld1.32   {W6, W5}, [RDATA]!;                           \
 156        vadd.u32  tmp0, W0, curK;                               \
 157 ARM_LE(vrev32.8  W7, W7;       )       /* big => little */     \
 158 ARM_LE(vrev32.8  W6, W6;       )       /* big => little */     \
 159        vadd.u32  tmp1, W7, curK;                               \
 160 ARM_LE(vrev32.8  W5, W5;       )       /* big => little */     \
 161        vadd.u32  tmp2, W6, curK;                               \
 162        vst1.32   {tmp0, tmp1}, [RWK]!;                         \
 163        vadd.u32  tmp3, W5, curK;                               \
 164        vst1.32   {tmp2, tmp3}, [RWK];                          \
 165
 166#define WPRECALC_00_15_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 167        vld1.32   {W0, W7}, [RDATA]!;                           \
 168
 169#define WPRECALC_00_15_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 170        add       RWK, sp, #(WK_offs(0));                       \
 171
 172#define WPRECALC_00_15_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 173 ARM_LE(vrev32.8  W0, W0;       )       /* big => little */     \
 174
 175#define WPRECALC_00_15_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 176        vld1.32   {W6, W5}, [RDATA]!;                           \
 177
 178#define WPRECALC_00_15_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 179        vadd.u32  tmp0, W0, curK;                               \
 180
 181#define WPRECALC_00_15_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 182 ARM_LE(vrev32.8  W7, W7;       )       /* big => little */     \
 183
 184#define WPRECALC_00_15_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 185 ARM_LE(vrev32.8  W6, W6;       )       /* big => little */     \
 186
 187#define WPRECALC_00_15_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 188        vadd.u32  tmp1, W7, curK;                               \
 189
 190#define WPRECALC_00_15_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 191 ARM_LE(vrev32.8  W5, W5;       )       /* big => little */     \
 192
 193#define WPRECALC_00_15_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 194        vadd.u32  tmp2, W6, curK;                               \
 195
 196#define WPRECALC_00_15_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 197        vst1.32   {tmp0, tmp1}, [RWK]!;                         \
 198
 199#define WPRECALC_00_15_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 200        vadd.u32  tmp3, W5, curK;                               \
 201
 202#define WPRECALC_00_15_12(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 203        vst1.32   {tmp2, tmp3}, [RWK];                          \
 204
 205
 206/********* Precalc macros for rounds 16-31 ************************************/
 207
 208#define WPRECALC_16_31_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 209        veor      tmp0, tmp0;                   \
 210        vext.8    W, W_m16, W_m12, #8;          \
 211
 212#define WPRECALC_16_31_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 213        add       RWK, sp, #(WK_offs(i));       \
 214        vext.8    tmp0, W_m04, tmp0, #4;        \
 215
 216#define WPRECALC_16_31_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 217        veor      tmp0, tmp0, W_m16;            \
 218        veor.32   W, W, W_m08;                  \
 219
 220#define WPRECALC_16_31_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 221        veor      tmp1, tmp1;                   \
 222        veor      W, W, tmp0;                   \
 223
 224#define WPRECALC_16_31_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 225        vshl.u32  tmp0, W, #1;                  \
 226
 227#define WPRECALC_16_31_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 228        vext.8    tmp1, tmp1, W, #(16-12);      \
 229        vshr.u32  W, W, #31;                    \
 230
 231#define WPRECALC_16_31_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 232        vorr      tmp0, tmp0, W;                \
 233        vshr.u32  W, tmp1, #30;                 \
 234
 235#define WPRECALC_16_31_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 236        vshl.u32  tmp1, tmp1, #2;               \
 237
 238#define WPRECALC_16_31_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 239        veor      tmp0, tmp0, W;                \
 240
 241#define WPRECALC_16_31_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 242        veor      W, tmp0, tmp1;                \
 243
 244#define WPRECALC_16_31_10(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 245        vadd.u32  tmp0, W, curK;                \
 246
 247#define WPRECALC_16_31_11(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 248        vst1.32   {tmp0}, [RWK];
 249
 250
 251/********* Precalc macros for rounds 32-79 ************************************/
 252
 253#define WPRECALC_32_79_0(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 254        veor W, W_m28; \
 255
 256#define WPRECALC_32_79_1(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 257        vext.8 tmp0, W_m08, W_m04, #8; \
 258
 259#define WPRECALC_32_79_2(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 260        veor W, W_m16; \
 261
 262#define WPRECALC_32_79_3(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 263        veor W, tmp0; \
 264
 265#define WPRECALC_32_79_4(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 266        add RWK, sp, #(WK_offs(i&~3)); \
 267
 268#define WPRECALC_32_79_5(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 269        vshl.u32 tmp1, W, #2; \
 270
 271#define WPRECALC_32_79_6(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 272        vshr.u32 tmp0, W, #30; \
 273
 274#define WPRECALC_32_79_7(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 275        vorr W, tmp0, tmp1; \
 276
 277#define WPRECALC_32_79_8(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 278        vadd.u32 tmp0, W, curK; \
 279
 280#define WPRECALC_32_79_9(i,W,W_m04,W_m08,W_m12,W_m16,W_m20,W_m24,W_m28) \
 281        vst1.32 {tmp0}, [RWK];
 282
 283
 284/*
 285 * Transform nblks*64 bytes (nblks*16 32-bit words) at DATA.
 286 *
 287 * unsigned int
 288 * sha1_transform_neon (void *ctx, const unsigned char *data,
 289 *                      unsigned int nblks)
 290 */
 291.align 3
 292ENTRY(sha1_transform_neon)
 293  /* input:
 294   *    r0: ctx, CTX
 295   *    r1: data (64*nblks bytes)
 296   *    r2: nblks
 297   */
 298
 299  cmp RNBLKS, #0;
 300  beq .Ldo_nothing;
 301
 302  push {r4-r12, lr};
 303  /*vpush {q4-q7};*/
 304
 305  adr RT3, .LK_VEC;
 306
 307  mov ROLDSTACK, sp;
 308
 309  /* Align stack. */
 310  sub RT0, sp, #(16*4);
 311  and RT0, #(~(16-1));
 312  mov sp, RT0;
 313
 314  vld1.32 {qK1-qK2}, [RT3]!; /* Load K1,K2 */
 315
 316  /* Get the values of the chaining variables. */
 317  ldm RSTATE, {_a-_e};
 318
 319  vld1.32 {qK3-qK4}, [RT3]; /* Load K3,K4 */
 320
 321#undef curK
 322#define curK qK1
 323  /* Precalc 0-15. */
 324  W_PRECALC_00_15();
 325
 326.Loop:
 327  /* Transform 0-15 + Precalc 16-31. */
 328  _R( _a, _b, _c, _d, _e, F1,  0,
 329      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 16,
 330      W4, W5, W6, W7, W0, _, _, _ );
 331  _R( _e, _a, _b, _c, _d, F1,  1,
 332      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 16,
 333      W4, W5, W6, W7, W0, _, _, _ );
 334  _R( _d, _e, _a, _b, _c, F1,  2,
 335      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 16,
 336      W4, W5, W6, W7, W0, _, _, _ );
 337  _R( _c, _d, _e, _a, _b, F1,  3,
 338      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,16,
 339      W4, W5, W6, W7, W0, _, _, _ );
 340
 341#undef curK
 342#define curK qK2
 343  _R( _b, _c, _d, _e, _a, F1,  4,
 344      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 20,
 345      W3, W4, W5, W6, W7, _, _, _ );
 346  _R( _a, _b, _c, _d, _e, F1,  5,
 347      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 20,
 348      W3, W4, W5, W6, W7, _, _, _ );
 349  _R( _e, _a, _b, _c, _d, F1,  6,
 350      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 20,
 351      W3, W4, W5, W6, W7, _, _, _ );
 352  _R( _d, _e, _a, _b, _c, F1,  7,
 353      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,20,
 354      W3, W4, W5, W6, W7, _, _, _ );
 355
 356  _R( _c, _d, _e, _a, _b, F1,  8,
 357      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 24,
 358      W2, W3, W4, W5, W6, _, _, _ );
 359  _R( _b, _c, _d, _e, _a, F1,  9,
 360      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 24,
 361      W2, W3, W4, W5, W6, _, _, _ );
 362  _R( _a, _b, _c, _d, _e, F1, 10,
 363      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 24,
 364      W2, W3, W4, W5, W6, _, _, _ );
 365  _R( _e, _a, _b, _c, _d, F1, 11,
 366      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,24,
 367      W2, W3, W4, W5, W6, _, _, _ );
 368
 369  _R( _d, _e, _a, _b, _c, F1, 12,
 370      WPRECALC_16_31_0, WPRECALC_16_31_1, WPRECALC_16_31_2, 28,
 371      W1, W2, W3, W4, W5, _, _, _ );
 372  _R( _c, _d, _e, _a, _b, F1, 13,
 373      WPRECALC_16_31_3, WPRECALC_16_31_4, WPRECALC_16_31_5, 28,
 374      W1, W2, W3, W4, W5, _, _, _ );
 375  _R( _b, _c, _d, _e, _a, F1, 14,
 376      WPRECALC_16_31_6, WPRECALC_16_31_7, WPRECALC_16_31_8, 28,
 377      W1, W2, W3, W4, W5, _, _, _ );
 378  _R( _a, _b, _c, _d, _e, F1, 15,
 379      WPRECALC_16_31_9, WPRECALC_16_31_10,WPRECALC_16_31_11,28,
 380      W1, W2, W3, W4, W5, _, _, _ );
 381
 382  /* Transform 16-63 + Precalc 32-79. */
 383  _R( _e, _a, _b, _c, _d, F1, 16,
 384      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 32,
 385      W0, W1, W2, W3, W4, W5, W6, W7);
 386  _R( _d, _e, _a, _b, _c, F1, 17,
 387      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 32,
 388      W0, W1, W2, W3, W4, W5, W6, W7);
 389  _R( _c, _d, _e, _a, _b, F1, 18,
 390      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 32,
 391      W0, W1, W2, W3, W4, W5, W6, W7);
 392  _R( _b, _c, _d, _e, _a, F1, 19,
 393      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 32,
 394      W0, W1, W2, W3, W4, W5, W6, W7);
 395
 396  _R( _a, _b, _c, _d, _e, F2, 20,
 397      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 36,
 398      W7, W0, W1, W2, W3, W4, W5, W6);
 399  _R( _e, _a, _b, _c, _d, F2, 21,
 400      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 36,
 401      W7, W0, W1, W2, W3, W4, W5, W6);
 402  _R( _d, _e, _a, _b, _c, F2, 22,
 403      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 36,
 404      W7, W0, W1, W2, W3, W4, W5, W6);
 405  _R( _c, _d, _e, _a, _b, F2, 23,
 406      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 36,
 407      W7, W0, W1, W2, W3, W4, W5, W6);
 408
 409#undef curK
 410#define curK qK3
 411  _R( _b, _c, _d, _e, _a, F2, 24,
 412      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 40,
 413      W6, W7, W0, W1, W2, W3, W4, W5);
 414  _R( _a, _b, _c, _d, _e, F2, 25,
 415      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 40,
 416      W6, W7, W0, W1, W2, W3, W4, W5);
 417  _R( _e, _a, _b, _c, _d, F2, 26,
 418      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 40,
 419      W6, W7, W0, W1, W2, W3, W4, W5);
 420  _R( _d, _e, _a, _b, _c, F2, 27,
 421      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 40,
 422      W6, W7, W0, W1, W2, W3, W4, W5);
 423
 424  _R( _c, _d, _e, _a, _b, F2, 28,
 425      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 44,
 426      W5, W6, W7, W0, W1, W2, W3, W4);
 427  _R( _b, _c, _d, _e, _a, F2, 29,
 428      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 44,
 429      W5, W6, W7, W0, W1, W2, W3, W4);
 430  _R( _a, _b, _c, _d, _e, F2, 30,
 431      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 44,
 432      W5, W6, W7, W0, W1, W2, W3, W4);
 433  _R( _e, _a, _b, _c, _d, F2, 31,
 434      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 44,
 435      W5, W6, W7, W0, W1, W2, W3, W4);
 436
 437  _R( _d, _e, _a, _b, _c, F2, 32,
 438      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 48,
 439      W4, W5, W6, W7, W0, W1, W2, W3);
 440  _R( _c, _d, _e, _a, _b, F2, 33,
 441      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 48,
 442      W4, W5, W6, W7, W0, W1, W2, W3);
 443  _R( _b, _c, _d, _e, _a, F2, 34,
 444      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 48,
 445      W4, W5, W6, W7, W0, W1, W2, W3);
 446  _R( _a, _b, _c, _d, _e, F2, 35,
 447      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 48,
 448      W4, W5, W6, W7, W0, W1, W2, W3);
 449
 450  _R( _e, _a, _b, _c, _d, F2, 36,
 451      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 52,
 452      W3, W4, W5, W6, W7, W0, W1, W2);
 453  _R( _d, _e, _a, _b, _c, F2, 37,
 454      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 52,
 455      W3, W4, W5, W6, W7, W0, W1, W2);
 456  _R( _c, _d, _e, _a, _b, F2, 38,
 457      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 52,
 458      W3, W4, W5, W6, W7, W0, W1, W2);
 459  _R( _b, _c, _d, _e, _a, F2, 39,
 460      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 52,
 461      W3, W4, W5, W6, W7, W0, W1, W2);
 462
 463  _R( _a, _b, _c, _d, _e, F3, 40,
 464      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 56,
 465      W2, W3, W4, W5, W6, W7, W0, W1);
 466  _R( _e, _a, _b, _c, _d, F3, 41,
 467      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 56,
 468      W2, W3, W4, W5, W6, W7, W0, W1);
 469  _R( _d, _e, _a, _b, _c, F3, 42,
 470      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 56,
 471      W2, W3, W4, W5, W6, W7, W0, W1);
 472  _R( _c, _d, _e, _a, _b, F3, 43,
 473      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 56,
 474      W2, W3, W4, W5, W6, W7, W0, W1);
 475
 476#undef curK
 477#define curK qK4
 478  _R( _b, _c, _d, _e, _a, F3, 44,
 479      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 60,
 480      W1, W2, W3, W4, W5, W6, W7, W0);
 481  _R( _a, _b, _c, _d, _e, F3, 45,
 482      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 60,
 483      W1, W2, W3, W4, W5, W6, W7, W0);
 484  _R( _e, _a, _b, _c, _d, F3, 46,
 485      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 60,
 486      W1, W2, W3, W4, W5, W6, W7, W0);
 487  _R( _d, _e, _a, _b, _c, F3, 47,
 488      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 60,
 489      W1, W2, W3, W4, W5, W6, W7, W0);
 490
 491  _R( _c, _d, _e, _a, _b, F3, 48,
 492      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 64,
 493      W0, W1, W2, W3, W4, W5, W6, W7);
 494  _R( _b, _c, _d, _e, _a, F3, 49,
 495      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 64,
 496      W0, W1, W2, W3, W4, W5, W6, W7);
 497  _R( _a, _b, _c, _d, _e, F3, 50,
 498      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 64,
 499      W0, W1, W2, W3, W4, W5, W6, W7);
 500  _R( _e, _a, _b, _c, _d, F3, 51,
 501      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 64,
 502      W0, W1, W2, W3, W4, W5, W6, W7);
 503
 504  _R( _d, _e, _a, _b, _c, F3, 52,
 505      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 68,
 506      W7, W0, W1, W2, W3, W4, W5, W6);
 507  _R( _c, _d, _e, _a, _b, F3, 53,
 508      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 68,
 509      W7, W0, W1, W2, W3, W4, W5, W6);
 510  _R( _b, _c, _d, _e, _a, F3, 54,
 511      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 68,
 512      W7, W0, W1, W2, W3, W4, W5, W6);
 513  _R( _a, _b, _c, _d, _e, F3, 55,
 514      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 68,
 515      W7, W0, W1, W2, W3, W4, W5, W6);
 516
 517  _R( _e, _a, _b, _c, _d, F3, 56,
 518      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 72,
 519      W6, W7, W0, W1, W2, W3, W4, W5);
 520  _R( _d, _e, _a, _b, _c, F3, 57,
 521      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 72,
 522      W6, W7, W0, W1, W2, W3, W4, W5);
 523  _R( _c, _d, _e, _a, _b, F3, 58,
 524      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 72,
 525      W6, W7, W0, W1, W2, W3, W4, W5);
 526  _R( _b, _c, _d, _e, _a, F3, 59,
 527      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 72,
 528      W6, W7, W0, W1, W2, W3, W4, W5);
 529
 530  subs RNBLKS, #1;
 531
 532  _R( _a, _b, _c, _d, _e, F4, 60,
 533      WPRECALC_32_79_0, WPRECALC_32_79_1, WPRECALC_32_79_2, 76,
 534      W5, W6, W7, W0, W1, W2, W3, W4);
 535  _R( _e, _a, _b, _c, _d, F4, 61,
 536      WPRECALC_32_79_3, WPRECALC_32_79_4, WPRECALC_32_79_5, 76,
 537      W5, W6, W7, W0, W1, W2, W3, W4);
 538  _R( _d, _e, _a, _b, _c, F4, 62,
 539      WPRECALC_32_79_6, dummy,            WPRECALC_32_79_7, 76,
 540      W5, W6, W7, W0, W1, W2, W3, W4);
 541  _R( _c, _d, _e, _a, _b, F4, 63,
 542      WPRECALC_32_79_8, dummy,            WPRECALC_32_79_9, 76,
 543      W5, W6, W7, W0, W1, W2, W3, W4);
 544
 545  beq .Lend;
 546
 547  /* Transform 64-79 + Precalc 0-15 of next block. */
 548#undef curK
 549#define curK qK1
 550  _R( _b, _c, _d, _e, _a, F4, 64,
 551      WPRECALC_00_15_0, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 552  _R( _a, _b, _c, _d, _e, F4, 65,
 553      WPRECALC_00_15_1, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 554  _R( _e, _a, _b, _c, _d, F4, 66,
 555      WPRECALC_00_15_2, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 556  _R( _d, _e, _a, _b, _c, F4, 67,
 557      WPRECALC_00_15_3, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 558
 559  _R( _c, _d, _e, _a, _b, F4, 68,
 560      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
 561  _R( _b, _c, _d, _e, _a, F4, 69,
 562      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
 563  _R( _a, _b, _c, _d, _e, F4, 70,
 564      WPRECALC_00_15_4, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 565  _R( _e, _a, _b, _c, _d, F4, 71,
 566      WPRECALC_00_15_5, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 567
 568  _R( _d, _e, _a, _b, _c, F4, 72,
 569      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
 570  _R( _c, _d, _e, _a, _b, F4, 73,
 571      dummy,            dummy, dummy, _, _, _, _, _, _, _, _, _ );
 572  _R( _b, _c, _d, _e, _a, F4, 74,
 573      WPRECALC_00_15_6, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 574  _R( _a, _b, _c, _d, _e, F4, 75,
 575      WPRECALC_00_15_7, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 576
 577  _R( _e, _a, _b, _c, _d, F4, 76,
 578      WPRECALC_00_15_8, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 579  _R( _d, _e, _a, _b, _c, F4, 77,
 580      WPRECALC_00_15_9, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 581  _R( _c, _d, _e, _a, _b, F4, 78,
 582      WPRECALC_00_15_10, dummy, dummy, _, _, _, _, _, _, _, _, _ );
 583  _R( _b, _c, _d, _e, _a, F4, 79,
 584      WPRECALC_00_15_11, dummy, WPRECALC_00_15_12, _, _, _, _, _, _, _, _, _ );
 585
 586  /* Update the chaining variables. */
 587  ldm RSTATE, {RT0-RT3};
 588  add _a, RT0;
 589  ldr RT0, [RSTATE, #state_h4];
 590  add _b, RT1;
 591  add _c, RT2;
 592  add _d, RT3;
 593  add _e, RT0;
 594  stm RSTATE, {_a-_e};
 595
 596  b .Loop;
 597
 598.Lend:
 599  /* Transform 64-79 */
 600  R( _b, _c, _d, _e, _a, F4, 64 );
 601  R( _a, _b, _c, _d, _e, F4, 65 );
 602  R( _e, _a, _b, _c, _d, F4, 66 );
 603  R( _d, _e, _a, _b, _c, F4, 67 );
 604  R( _c, _d, _e, _a, _b, F4, 68 );
 605  R( _b, _c, _d, _e, _a, F4, 69 );
 606  R( _a, _b, _c, _d, _e, F4, 70 );
 607  R( _e, _a, _b, _c, _d, F4, 71 );
 608  R( _d, _e, _a, _b, _c, F4, 72 );
 609  R( _c, _d, _e, _a, _b, F4, 73 );
 610  R( _b, _c, _d, _e, _a, F4, 74 );
 611  R( _a, _b, _c, _d, _e, F4, 75 );
 612  R( _e, _a, _b, _c, _d, F4, 76 );
 613  R( _d, _e, _a, _b, _c, F4, 77 );
 614  R( _c, _d, _e, _a, _b, F4, 78 );
 615  R( _b, _c, _d, _e, _a, F4, 79 );
 616
 617  mov sp, ROLDSTACK;
 618
 619  /* Update the chaining variables. */
 620  ldm RSTATE, {RT0-RT3};
 621  add _a, RT0;
 622  ldr RT0, [RSTATE, #state_h4];
 623  add _b, RT1;
 624  add _c, RT2;
 625  add _d, RT3;
 626  /*vpop {q4-q7};*/
 627  add _e, RT0;
 628  stm RSTATE, {_a-_e};
 629
 630  pop {r4-r12, pc};
 631
 632.Ldo_nothing:
 633  bx lr
 634ENDPROC(sha1_transform_neon)
 635