LXR linux/arch/x86/crypto/nh-sse2-x86

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
   4 *
   5 * Copyright 2018 Google LLC
   6 *
   7 * Author: Eric Biggers <ebiggers@google.com>
   8 */
   9
  10#include <linux/linkage.h>
  11
  12#define         PASS0_SUMS      %xmm0
  13#define         PASS1_SUMS      %xmm1
  14#define         PASS2_SUMS      %xmm2
  15#define         PASS3_SUMS      %xmm3
  16#define         K0              %xmm4
  17#define         K1              %xmm5
  18#define         K2              %xmm6
  19#define         K3              %xmm7
  20#define         T0              %xmm8
  21#define         T1              %xmm9
  22#define         T2              %xmm10
  23#define         T3              %xmm11
  24#define         T4              %xmm12
  25#define         T5              %xmm13
  26#define         T6              %xmm14
  27#define         T7              %xmm15
  28#define         KEY             %rdi
  29#define         MESSAGE         %rsi
  30#define         MESSAGE_LEN     %rdx
  31#define         HASH            %rcx
  32
  33.macro _nh_stride       k0, k1, k2, k3, offset
  34
  35        // Load next message stride
  36        movdqu          \offset(MESSAGE), T1
  37
  38        // Load next key stride
  39        movdqu          \offset(KEY), \k3
  40
  41        // Add message words to key words
  42        movdqa          T1, T2
  43        movdqa          T1, T3
  44        paddd           T1, \k0    // reuse k0 to avoid a move
  45        paddd           \k1, T1
  46        paddd           \k2, T2
  47        paddd           \k3, T3
  48
  49        // Multiply 32x32 => 64 and accumulate
  50        pshufd          $0x10, \k0, T4
  51        pshufd          $0x32, \k0, \k0
  52        pshufd          $0x10, T1, T5
  53        pshufd          $0x32, T1, T1
  54        pshufd          $0x10, T2, T6
  55        pshufd          $0x32, T2, T2
  56        pshufd          $0x10, T3, T7
  57        pshufd          $0x32, T3, T3
  58        pmuludq         T4, \k0
  59        pmuludq         T5, T1
  60        pmuludq         T6, T2
  61        pmuludq         T7, T3
  62        paddq           \k0, PASS0_SUMS
  63        paddq           T1, PASS1_SUMS
  64        paddq           T2, PASS2_SUMS
  65        paddq           T3, PASS3_SUMS
  66.endm
  67
  68/*
  69 * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
  70 *              u8 hash[NH_HASH_BYTES])
  71 *
  72 * It's guaranteed that message_len % 16 == 0.
  73 */
  74ENTRY(nh_sse2)
  75
  76        movdqu          0x00(KEY), K0
  77        movdqu          0x10(KEY), K1
  78        movdqu          0x20(KEY), K2
  79        add             $0x30, KEY
  80        pxor            PASS0_SUMS, PASS0_SUMS
  81        pxor            PASS1_SUMS, PASS1_SUMS
  82        pxor            PASS2_SUMS, PASS2_SUMS
  83        pxor            PASS3_SUMS, PASS3_SUMS
  84
  85        sub             $0x40, MESSAGE_LEN
  86        jl              .Lloop4_done
  87.Lloop4:
  88        _nh_stride      K0, K1, K2, K3, 0x00
  89        _nh_stride      K1, K2, K3, K0, 0x10
  90        _nh_stride      K2, K3, K0, K1, 0x20
  91        _nh_stride      K3, K0, K1, K2, 0x30
  92        add             $0x40, KEY
  93        add             $0x40, MESSAGE
  94        sub             $0x40, MESSAGE_LEN
  95        jge             .Lloop4
  96
  97.Lloop4_done:
  98        and             $0x3f, MESSAGE_LEN
  99        jz              .Ldone
 100        _nh_stride      K0, K1, K2, K3, 0x00
 101
 102        sub             $0x10, MESSAGE_LEN
 103        jz              .Ldone
 104        _nh_stride      K1, K2, K3, K0, 0x10
 105
 106        sub             $0x10, MESSAGE_LEN
 107        jz              .Ldone
 108        _nh_stride      K2, K3, K0, K1, 0x20
 109
 110.Ldone:
 111        // Sum the accumulators for each pass, then store the sums to 'hash'
 112        movdqa          PASS0_SUMS, T0
 113        movdqa          PASS2_SUMS, T1
 114        punpcklqdq      PASS1_SUMS, T0          // => (PASS0_SUM_A PASS1_SUM_A)
 115        punpcklqdq      PASS3_SUMS, T1          // => (PASS2_SUM_A PASS3_SUM_A)
 116        punpckhqdq      PASS1_SUMS, PASS0_SUMS  // => (PASS0_SUM_B PASS1_SUM_B)
 117        punpckhqdq      PASS3_SUMS, PASS2_SUMS  // => (PASS2_SUM_B PASS3_SUM_B)
 118        paddq           PASS0_SUMS, T0
 119        paddq           PASS2_SUMS, T1
 120        movdqu          T0, 0x00(HASH)
 121        movdqu          T1, 0x10(HASH)
 122        ret
 123ENDPROC(nh_sse2)
 124