LXR linux/arch/x86/crypto/nhpoly1305-avx2-glue.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
   4 * (AVX2 accelerated version)
   5 *
   6 * Copyright 2018 Google LLC
   7 */
   8
   9#include <crypto/internal/hash.h>
  10#include <crypto/internal/simd.h>
  11#include <crypto/nhpoly1305.h>
  12#include <linux/module.h>
  13#include <linux/sizes.h>
  14#include <asm/simd.h>
  15
  16asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
  17                        u8 hash[NH_HASH_BYTES]);
  18
  19/* wrapper to avoid indirect call to assembly, which doesn't work with CFI */
  20static void _nh_avx2(const u32 *key, const u8 *message, size_t message_len,
  21                     __le64 hash[NH_NUM_PASSES])
  22{
  23        nh_avx2(key, message, message_len, (u8 *)hash);
  24}
  25
  26static int nhpoly1305_avx2_update(struct shash_desc *desc,
  27                                  const u8 *src, unsigned int srclen)
  28{
  29        if (srclen < 64 || !crypto_simd_usable())
  30                return crypto_nhpoly1305_update(desc, src, srclen);
  31
  32        do {
  33                unsigned int n = min_t(unsigned int, srclen, SZ_4K);
  34
  35                kernel_fpu_begin();
  36                crypto_nhpoly1305_update_helper(desc, src, n, _nh_avx2);
  37                kernel_fpu_end();
  38                src += n;
  39                srclen -= n;
  40        } while (srclen);
  41        return 0;
  42}
  43
  44static struct shash_alg nhpoly1305_alg = {
  45        .base.cra_name          = "nhpoly1305",
  46        .base.cra_driver_name   = "nhpoly1305-avx2",
  47        .base.cra_priority      = 300,
  48        .base.cra_ctxsize       = sizeof(struct nhpoly1305_key),
  49        .base.cra_module        = THIS_MODULE,
  50        .digestsize             = POLY1305_DIGEST_SIZE,
  51        .init                   = crypto_nhpoly1305_init,
  52        .update                 = nhpoly1305_avx2_update,
  53        .final                  = crypto_nhpoly1305_final,
  54        .setkey                 = crypto_nhpoly1305_setkey,
  55        .descsize               = sizeof(struct nhpoly1305_state),
  56};
  57
  58static int __init nhpoly1305_mod_init(void)
  59{
  60        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
  61            !boot_cpu_has(X86_FEATURE_OSXSAVE))
  62                return -ENODEV;
  63
  64        return crypto_register_shash(&nhpoly1305_alg);
  65}
  66
  67static void __exit nhpoly1305_mod_exit(void)
  68{
  69        crypto_unregister_shash(&nhpoly1305_alg);
  70}
  71
  72module_init(nhpoly1305_mod_init);
  73module_exit(nhpoly1305_mod_exit);
  74
  75MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");
  76MODULE_LICENSE("GPL v2");
  77MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
  78MODULE_ALIAS_CRYPTO("nhpoly1305");
  79MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");
  80