LXR linux/arch/x86/crypto/sha1_ssse3

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 * This is a SIMD SHA-1 implementation. It requires the Intel(R) Supplemental
   4 * SSE3 instruction set extensions introduced in Intel Core Microarchitecture
   5 * processors. CPUs supporting Intel(R) AVX extensions will get an additional
   6 * boost.
   7 *
   8 * This work was inspired by the vectorized implementation of Dean Gaudet.
   9 * Additional information on it can be found at:
  10 *    http://www.arctic.org/~dean/crypto/sha1.html
  11 *
  12 * It was improved upon with more efficient vectorization of the message
  13 * scheduling. This implementation has also been optimized for all current and
  14 * several future generations of Intel CPUs.
  15 *
  16 * See this article for more information about the implementation details:
  17 *   http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/
  18 *
  19 * Copyright (C) 2010, Intel Corp.
  20 *   Authors: Maxim Locktyukhin <maxim.locktyukhin@intel.com>
  21 *            Ronen Zohar <ronen.zohar@intel.com>
  22 *
  23 * Converted to AT&T syntax and adapted for inclusion in the Linux kernel:
  24 *   Author: Mathias Krause <minipli@googlemail.com>
  25 */
  26
  27#include <linux/linkage.h>
  28
  29#define CTX     %rdi    // arg1
  30#define BUF     %rsi    // arg2
  31#define CNT     %rdx    // arg3
  32
  33#define REG_A   %ecx
  34#define REG_B   %esi
  35#define REG_C   %edi
  36#define REG_D   %r12d
  37#define REG_E   %edx
  38
  39#define REG_T1  %eax
  40#define REG_T2  %ebx
  41
  42#define K_BASE          %r8
  43#define HASH_PTR        %r9
  44#define BUFFER_PTR      %r10
  45#define BUFFER_END      %r11
  46
  47#define W_TMP1  %xmm0
  48#define W_TMP2  %xmm9
  49
  50#define W0      %xmm1
  51#define W4      %xmm2
  52#define W8      %xmm3
  53#define W12     %xmm4
  54#define W16     %xmm5
  55#define W20     %xmm6
  56#define W24     %xmm7
  57#define W28     %xmm8
  58
  59#define XMM_SHUFB_BSWAP %xmm10
  60
  61/* we keep window of 64 w[i]+K pre-calculated values in a circular buffer */
  62#define WK(t)   (((t) & 15) * 4)(%rsp)
  63#define W_PRECALC_AHEAD 16
  64
  65/*
  66 * This macro implements the SHA-1 function's body for single 64-byte block
  67 * param: function's name
  68 */
  69.macro SHA1_VECTOR_ASM  name
  70        SYM_FUNC_START(\name)
  71
  72        push    %rbx
  73        push    %r12
  74        push    %rbp
  75        mov     %rsp, %rbp
  76
  77        sub     $64, %rsp               # allocate workspace
  78        and     $~15, %rsp              # align stack
  79
  80        mov     CTX, HASH_PTR
  81        mov     BUF, BUFFER_PTR
  82
  83        shl     $6, CNT                 # multiply by 64
  84        add     BUF, CNT
  85        mov     CNT, BUFFER_END
  86
  87        lea     K_XMM_AR(%rip), K_BASE
  88        xmm_mov BSWAP_SHUFB_CTL(%rip), XMM_SHUFB_BSWAP
  89
  90        SHA1_PIPELINED_MAIN_BODY
  91
  92        # cleanup workspace
  93        mov     $8, %ecx
  94        mov     %rsp, %rdi
  95        xor     %eax, %eax
  96        rep stosq
  97
  98        mov     %rbp, %rsp              # deallocate workspace
  99        pop     %rbp
 100        pop     %r12
 101        pop     %rbx
 102        ret
 103
 104        SYM_FUNC_END(\name)
 105.endm
 106
 107/*
 108 * This macro implements 80 rounds of SHA-1 for one 64-byte block
 109 */
 110.macro SHA1_PIPELINED_MAIN_BODY
 111        INIT_REGALLOC
 112
 113        mov       (HASH_PTR), A
 114        mov      4(HASH_PTR), B
 115        mov      8(HASH_PTR), C
 116        mov     12(HASH_PTR), D
 117        mov     16(HASH_PTR), E
 118
 119  .set i, 0
 120  .rept W_PRECALC_AHEAD
 121        W_PRECALC i
 122    .set i, (i+1)
 123  .endr
 124
 125.align 4
 1261:
 127        RR F1,A,B,C,D,E,0
 128        RR F1,D,E,A,B,C,2
 129        RR F1,B,C,D,E,A,4
 130        RR F1,E,A,B,C,D,6
 131        RR F1,C,D,E,A,B,8
 132
 133        RR F1,A,B,C,D,E,10
 134        RR F1,D,E,A,B,C,12
 135        RR F1,B,C,D,E,A,14
 136        RR F1,E,A,B,C,D,16
 137        RR F1,C,D,E,A,B,18
 138
 139        RR F2,A,B,C,D,E,20
 140        RR F2,D,E,A,B,C,22
 141        RR F2,B,C,D,E,A,24
 142        RR F2,E,A,B,C,D,26
 143        RR F2,C,D,E,A,B,28
 144
 145        RR F2,A,B,C,D,E,30
 146        RR F2,D,E,A,B,C,32
 147        RR F2,B,C,D,E,A,34
 148        RR F2,E,A,B,C,D,36
 149        RR F2,C,D,E,A,B,38
 150
 151        RR F3,A,B,C,D,E,40
 152        RR F3,D,E,A,B,C,42
 153        RR F3,B,C,D,E,A,44
 154        RR F3,E,A,B,C,D,46
 155        RR F3,C,D,E,A,B,48
 156
 157        RR F3,A,B,C,D,E,50
 158        RR F3,D,E,A,B,C,52
 159        RR F3,B,C,D,E,A,54
 160        RR F3,E,A,B,C,D,56
 161        RR F3,C,D,E,A,B,58
 162
 163        add     $64, BUFFER_PTR         # move to the next 64-byte block
 164        cmp     BUFFER_END, BUFFER_PTR  # if the current is the last one use
 165        cmovae  K_BASE, BUFFER_PTR      # dummy source to avoid buffer overrun
 166
 167        RR F4,A,B,C,D,E,60
 168        RR F4,D,E,A,B,C,62
 169        RR F4,B,C,D,E,A,64
 170        RR F4,E,A,B,C,D,66
 171        RR F4,C,D,E,A,B,68
 172
 173        RR F4,A,B,C,D,E,70
 174        RR F4,D,E,A,B,C,72
 175        RR F4,B,C,D,E,A,74
 176        RR F4,E,A,B,C,D,76
 177        RR F4,C,D,E,A,B,78
 178
 179        UPDATE_HASH   (HASH_PTR), A
 180        UPDATE_HASH  4(HASH_PTR), B
 181        UPDATE_HASH  8(HASH_PTR), C
 182        UPDATE_HASH 12(HASH_PTR), D
 183        UPDATE_HASH 16(HASH_PTR), E
 184
 185        RESTORE_RENAMED_REGS
 186        cmp     K_BASE, BUFFER_PTR      # K_BASE means, we reached the end
 187        jne     1b
 188.endm
 189
 190.macro INIT_REGALLOC
 191  .set A, REG_A
 192  .set B, REG_B
 193  .set C, REG_C
 194  .set D, REG_D
 195  .set E, REG_E
 196  .set T1, REG_T1
 197  .set T2, REG_T2
 198.endm
 199
 200.macro RESTORE_RENAMED_REGS
 201        # order is important (REG_C is where it should be)
 202        mov     B, REG_B
 203        mov     D, REG_D
 204        mov     A, REG_A
 205        mov     E, REG_E
 206.endm
 207
 208.macro SWAP_REG_NAMES  a, b
 209  .set _T, \a
 210  .set \a, \b
 211  .set \b, _T
 212.endm
 213
 214.macro F1  b, c, d
 215        mov     \c, T1
 216        SWAP_REG_NAMES \c, T1
 217        xor     \d, T1
 218        and     \b, T1
 219        xor     \d, T1
 220.endm
 221
 222.macro F2  b, c, d
 223        mov     \d, T1
 224        SWAP_REG_NAMES \d, T1
 225        xor     \c, T1
 226        xor     \b, T1
 227.endm
 228
 229.macro F3  b, c ,d
 230        mov     \c, T1
 231        SWAP_REG_NAMES \c, T1
 232        mov     \b, T2
 233        or      \b, T1
 234        and     \c, T2
 235        and     \d, T1
 236        or      T2, T1
 237.endm
 238
 239.macro F4  b, c, d
 240        F2 \b, \c, \d
 241.endm
 242
 243.macro UPDATE_HASH  hash, val
 244        add     \hash, \val
 245        mov     \val, \hash
 246.endm
 247
 248/*
 249 * RR does two rounds of SHA-1 back to back with W[] pre-calc
 250 *   t1 = F(b, c, d);   e += w(i)
 251 *   e += t1;           b <<= 30;   d  += w(i+1);
 252 *   t1 = F(a, b, c);
 253 *   d += t1;           a <<= 5;
 254 *   e += a;
 255 *   t1 = e;            a >>= 7;
 256 *   t1 <<= 5;
 257 *   d += t1;
 258 */
 259.macro RR  F, a, b, c, d, e, round
 260        add     WK(\round), \e
 261        \F   \b, \c, \d         # t1 = F(b, c, d);
 262        W_PRECALC (\round + W_PRECALC_AHEAD)
 263        rol     $30, \b
 264        add     T1, \e
 265        add     WK(\round + 1), \d
 266
 267        \F   \a, \b, \c
 268        W_PRECALC (\round + W_PRECALC_AHEAD + 1)
 269        rol     $5, \a
 270        add     \a, \e
 271        add     T1, \d
 272        ror     $7, \a          # (a <<r 5) >>r 7) => a <<r 30)
 273
 274        mov     \e, T1
 275        SWAP_REG_NAMES \e, T1
 276
 277        rol     $5, T1
 278        add     T1, \d
 279
 280        # write:  \a, \b
 281        # rotate: \a<=\d, \b<=\e, \c<=\a, \d<=\b, \e<=\c
 282.endm
 283
 284.macro W_PRECALC  r
 285  .set i, \r
 286
 287  .if (i < 20)
 288    .set K_XMM, 0
 289  .elseif (i < 40)
 290    .set K_XMM, 16
 291  .elseif (i < 60)
 292    .set K_XMM, 32
 293  .elseif (i < 80)
 294    .set K_XMM, 48
 295  .endif
 296
 297  .if ((i < 16) || ((i >= 80) && (i < (80 + W_PRECALC_AHEAD))))
 298    .set i, ((\r) % 80)     # pre-compute for the next iteration
 299    .if (i == 0)
 300        W_PRECALC_RESET
 301    .endif
 302        W_PRECALC_00_15
 303  .elseif (i<32)
 304        W_PRECALC_16_31
 305  .elseif (i < 80)   // rounds 32-79
 306        W_PRECALC_32_79
 307  .endif
 308.endm
 309
 310.macro W_PRECALC_RESET
 311  .set W,          W0
 312  .set W_minus_04, W4
 313  .set W_minus_08, W8
 314  .set W_minus_12, W12
 315  .set W_minus_16, W16
 316  .set W_minus_20, W20
 317  .set W_minus_24, W24
 318  .set W_minus_28, W28
 319  .set W_minus_32, W
 320.endm
 321
 322.macro W_PRECALC_ROTATE
 323  .set W_minus_32, W_minus_28
 324  .set W_minus_28, W_minus_24
 325  .set W_minus_24, W_minus_20
 326  .set W_minus_20, W_minus_16
 327  .set W_minus_16, W_minus_12
 328  .set W_minus_12, W_minus_08
 329  .set W_minus_08, W_minus_04
 330  .set W_minus_04, W
 331  .set W,          W_minus_32
 332.endm
 333
 334.macro W_PRECALC_SSSE3
 335
 336.macro W_PRECALC_00_15
 337        W_PRECALC_00_15_SSSE3
 338.endm
 339.macro W_PRECALC_16_31
 340        W_PRECALC_16_31_SSSE3
 341.endm
 342.macro W_PRECALC_32_79
 343        W_PRECALC_32_79_SSSE3
 344.endm
 345
 346/* message scheduling pre-compute for rounds 0-15 */
 347.macro W_PRECALC_00_15_SSSE3
 348  .if ((i & 3) == 0)
 349        movdqu  (i*4)(BUFFER_PTR), W_TMP1
 350  .elseif ((i & 3) == 1)
 351        pshufb  XMM_SHUFB_BSWAP, W_TMP1
 352        movdqa  W_TMP1, W
 353  .elseif ((i & 3) == 2)
 354        paddd   (K_BASE), W_TMP1
 355  .elseif ((i & 3) == 3)
 356        movdqa  W_TMP1, WK(i&~3)
 357        W_PRECALC_ROTATE
 358  .endif
 359.endm
 360
 361/* message scheduling pre-compute for rounds 16-31
 362 *
 363 * - calculating last 32 w[i] values in 8 XMM registers
 364 * - pre-calculate K+w[i] values and store to mem, for later load by ALU add
 365 *   instruction
 366 *
 367 * some "heavy-lifting" vectorization for rounds 16-31 due to w[i]->w[i-3]
 368 * dependency, but improves for 32-79
 369 */
 370.macro W_PRECALC_16_31_SSSE3
 371  # blended scheduling of vector and scalar instruction streams, one 4-wide
 372  # vector iteration / 4 scalar rounds
 373  .if ((i & 3) == 0)
 374        movdqa  W_minus_12, W
 375        palignr $8, W_minus_16, W       # w[i-14]
 376        movdqa  W_minus_04, W_TMP1
 377        psrldq  $4, W_TMP1              # w[i-3]
 378        pxor    W_minus_08, W
 379  .elseif ((i & 3) == 1)
 380        pxor    W_minus_16, W_TMP1
 381        pxor    W_TMP1, W
 382        movdqa  W, W_TMP2
 383        movdqa  W, W_TMP1
 384        pslldq  $12, W_TMP2
 385  .elseif ((i & 3) == 2)
 386        psrld   $31, W
 387        pslld   $1, W_TMP1
 388        por     W, W_TMP1
 389        movdqa  W_TMP2, W
 390        psrld   $30, W_TMP2
 391        pslld   $2, W
 392  .elseif ((i & 3) == 3)
 393        pxor    W, W_TMP1
 394        pxor    W_TMP2, W_TMP1
 395        movdqa  W_TMP1, W
 396        paddd   K_XMM(K_BASE), W_TMP1
 397        movdqa  W_TMP1, WK(i&~3)
 398        W_PRECALC_ROTATE
 399  .endif
 400.endm
 401
 402/* message scheduling pre-compute for rounds 32-79
 403 *
 404 * in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
 405 * instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
 406 * allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
 407 */
 408.macro W_PRECALC_32_79_SSSE3
 409  .if ((i & 3) == 0)
 410        movdqa  W_minus_04, W_TMP1
 411        pxor    W_minus_28, W           # W is W_minus_32 before xor
 412        palignr $8, W_minus_08, W_TMP1
 413  .elseif ((i & 3) == 1)
 414        pxor    W_minus_16, W
 415        pxor    W_TMP1, W
 416        movdqa  W, W_TMP1
 417  .elseif ((i & 3) == 2)
 418        psrld   $30, W
 419        pslld   $2, W_TMP1
 420        por     W, W_TMP1
 421  .elseif ((i & 3) == 3)
 422        movdqa  W_TMP1, W
 423        paddd   K_XMM(K_BASE), W_TMP1
 424        movdqa  W_TMP1, WK(i&~3)
 425        W_PRECALC_ROTATE
 426  .endif
 427.endm
 428
 429.endm           // W_PRECALC_SSSE3
 430
 431
 432#define K1      0x5a827999
 433#define K2      0x6ed9eba1
 434#define K3      0x8f1bbcdc
 435#define K4      0xca62c1d6
 436
 437.section .rodata
 438.align 16
 439
 440K_XMM_AR:
 441        .long K1, K1, K1, K1
 442        .long K2, K2, K2, K2
 443        .long K3, K3, K3, K3
 444        .long K4, K4, K4, K4
 445
 446BSWAP_SHUFB_CTL:
 447        .long 0x00010203
 448        .long 0x04050607
 449        .long 0x08090a0b
 450        .long 0x0c0d0e0f
 451
 452
 453.section .text
 454
 455W_PRECALC_SSSE3
 456.macro xmm_mov a, b
 457        movdqu  \a,\b
 458.endm
 459
 460/*
 461 * SSSE3 optimized implementation:
 462 *
 463 * extern "C" void sha1_transform_ssse3(struct sha1_state *state,
 464 *                                      const u8 *data, int blocks);
 465 *
 466 * Note that struct sha1_state is assumed to begin with u32 state[5].
 467 */
 468SHA1_VECTOR_ASM     sha1_transform_ssse3
 469
 470.macro W_PRECALC_AVX
 471
 472.purgem W_PRECALC_00_15
 473.macro  W_PRECALC_00_15
 474    W_PRECALC_00_15_AVX
 475.endm
 476.purgem W_PRECALC_16_31
 477.macro  W_PRECALC_16_31
 478    W_PRECALC_16_31_AVX
 479.endm
 480.purgem W_PRECALC_32_79
 481.macro  W_PRECALC_32_79
 482    W_PRECALC_32_79_AVX
 483.endm
 484
 485.macro W_PRECALC_00_15_AVX
 486  .if ((i & 3) == 0)
 487        vmovdqu (i*4)(BUFFER_PTR), W_TMP1
 488  .elseif ((i & 3) == 1)
 489        vpshufb XMM_SHUFB_BSWAP, W_TMP1, W
 490  .elseif ((i & 3) == 2)
 491        vpaddd  (K_BASE), W, W_TMP1
 492  .elseif ((i & 3) == 3)
 493        vmovdqa W_TMP1, WK(i&~3)
 494        W_PRECALC_ROTATE
 495  .endif
 496.endm
 497
 498.macro W_PRECALC_16_31_AVX
 499  .if ((i & 3) == 0)
 500        vpalignr $8, W_minus_16, W_minus_12, W  # w[i-14]
 501        vpsrldq $4, W_minus_04, W_TMP1          # w[i-3]
 502        vpxor   W_minus_08, W, W
 503        vpxor   W_minus_16, W_TMP1, W_TMP1
 504  .elseif ((i & 3) == 1)
 505        vpxor   W_TMP1, W, W
 506        vpslldq $12, W, W_TMP2
 507        vpslld  $1, W, W_TMP1
 508  .elseif ((i & 3) == 2)
 509        vpsrld  $31, W, W
 510        vpor    W, W_TMP1, W_TMP1
 511        vpslld  $2, W_TMP2, W
 512        vpsrld  $30, W_TMP2, W_TMP2
 513  .elseif ((i & 3) == 3)
 514        vpxor   W, W_TMP1, W_TMP1
 515        vpxor   W_TMP2, W_TMP1, W
 516        vpaddd  K_XMM(K_BASE), W, W_TMP1
 517        vmovdqu W_TMP1, WK(i&~3)
 518        W_PRECALC_ROTATE
 519  .endif
 520.endm
 521
 522.macro W_PRECALC_32_79_AVX
 523  .if ((i & 3) == 0)
 524        vpalignr $8, W_minus_08, W_minus_04, W_TMP1
 525        vpxor   W_minus_28, W, W                # W is W_minus_32 before xor
 526  .elseif ((i & 3) == 1)
 527        vpxor   W_minus_16, W_TMP1, W_TMP1
 528        vpxor   W_TMP1, W, W
 529  .elseif ((i & 3) == 2)
 530        vpslld  $2, W, W_TMP1
 531        vpsrld  $30, W, W
 532        vpor    W, W_TMP1, W
 533  .elseif ((i & 3) == 3)
 534        vpaddd  K_XMM(K_BASE), W, W_TMP1
 535        vmovdqu W_TMP1, WK(i&~3)
 536        W_PRECALC_ROTATE
 537  .endif
 538.endm
 539
 540.endm    // W_PRECALC_AVX
 541
 542W_PRECALC_AVX
 543.purgem xmm_mov
 544.macro xmm_mov a, b
 545        vmovdqu \a,\b
 546.endm
 547
 548
 549/* AVX optimized implementation:
 550 *  extern "C" void sha1_transform_avx(struct sha1_state *state,
 551 *                                     const u8 *data, int blocks);
 552 */
 553SHA1_VECTOR_ASM     sha1_transform_avx
 554