LXR linux/arch/arm64/crypto/nhpoly1305-neon-glue.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
   4 * (ARM64 NEON accelerated version)
   5 *
   6 * Copyright 2018 Google LLC
   7 */
   8
   9#include <asm/neon.h>
  10#include <asm/simd.h>
  11#include <crypto/internal/hash.h>
  12#include <crypto/internal/simd.h>
  13#include <crypto/nhpoly1305.h>
  14#include <linux/module.h>
  15
  16asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
  17                        u8 hash[NH_HASH_BYTES]);
  18
  19/* wrapper to avoid indirect call to assembly, which doesn't work with CFI */
  20static void _nh_neon(const u32 *key, const u8 *message, size_t message_len,
  21                     __le64 hash[NH_NUM_PASSES])
  22{
  23        nh_neon(key, message, message_len, (u8 *)hash);
  24}
  25
  26static int nhpoly1305_neon_update(struct shash_desc *desc,
  27                                  const u8 *src, unsigned int srclen)
  28{
  29        if (srclen < 64 || !crypto_simd_usable())
  30                return crypto_nhpoly1305_update(desc, src, srclen);
  31
  32        do {
  33                unsigned int n = min_t(unsigned int, srclen, SZ_4K);
  34
  35                kernel_neon_begin();
  36                crypto_nhpoly1305_update_helper(desc, src, n, _nh_neon);
  37                kernel_neon_end();
  38                src += n;
  39                srclen -= n;
  40        } while (srclen);
  41        return 0;
  42}
  43
  44static struct shash_alg nhpoly1305_alg = {
  45        .base.cra_name          = "nhpoly1305",
  46        .base.cra_driver_name   = "nhpoly1305-neon",
  47        .base.cra_priority      = 200,
  48        .base.cra_ctxsize       = sizeof(struct nhpoly1305_key),
  49        .base.cra_module        = THIS_MODULE,
  50        .digestsize             = POLY1305_DIGEST_SIZE,
  51        .init                   = crypto_nhpoly1305_init,
  52        .update                 = nhpoly1305_neon_update,
  53        .final                  = crypto_nhpoly1305_final,
  54        .setkey                 = crypto_nhpoly1305_setkey,
  55        .descsize               = sizeof(struct nhpoly1305_state),
  56};
  57
  58static int __init nhpoly1305_mod_init(void)
  59{
  60        if (!cpu_have_named_feature(ASIMD))
  61                return -ENODEV;
  62
  63        return crypto_register_shash(&nhpoly1305_alg);
  64}
  65
  66static void __exit nhpoly1305_mod_exit(void)
  67{
  68        crypto_unregister_shash(&nhpoly1305_alg);
  69}
  70
  71module_init(nhpoly1305_mod_init);
  72module_exit(nhpoly1305_mod_exit);
  73
  74MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
  75MODULE_LICENSE("GPL v2");
  76MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
  77MODULE_ALIAS_CRYPTO("nhpoly1305");
  78MODULE_ALIAS_CRYPTO("nhpoly1305-neon");
  79