LXR linux/arch/arm64/crypto/sha256-glue.c

   1/*
   2 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
   3 *
   4 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License as published by the Free
   8 * Software Foundation; either version 2 of the License, or (at your option)
   9 * any later version.
  10 *
  11 */
  12
  13#include <asm/hwcap.h>
  14#include <asm/neon.h>
  15#include <asm/simd.h>
  16#include <crypto/internal/hash.h>
  17#include <crypto/sha.h>
  18#include <crypto/sha256_base.h>
  19#include <linux/cryptohash.h>
  20#include <linux/types.h>
  21#include <linux/string.h>
  22
  23MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
  24MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
  25MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  26MODULE_LICENSE("GPL v2");
  27MODULE_ALIAS_CRYPTO("sha224");
  28MODULE_ALIAS_CRYPTO("sha256");
  29
  30asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
  31                                        unsigned int num_blks);
  32EXPORT_SYMBOL(sha256_block_data_order);
  33
  34asmlinkage void sha256_block_neon(u32 *digest, const void *data,
  35                                  unsigned int num_blks);
  36
  37static int sha256_update(struct shash_desc *desc, const u8 *data,
  38                         unsigned int len)
  39{
  40        return sha256_base_do_update(desc, data, len,
  41                                (sha256_block_fn *)sha256_block_data_order);
  42}
  43
  44static int sha256_finup(struct shash_desc *desc, const u8 *data,
  45                        unsigned int len, u8 *out)
  46{
  47        if (len)
  48                sha256_base_do_update(desc, data, len,
  49                                (sha256_block_fn *)sha256_block_data_order);
  50        sha256_base_do_finalize(desc,
  51                                (sha256_block_fn *)sha256_block_data_order);
  52
  53        return sha256_base_finish(desc, out);
  54}
  55
  56static int sha256_final(struct shash_desc *desc, u8 *out)
  57{
  58        return sha256_finup(desc, NULL, 0, out);
  59}
  60
  61static struct shash_alg algs[] = { {
  62        .digestsize             = SHA256_DIGEST_SIZE,
  63        .init                   = sha256_base_init,
  64        .update                 = sha256_update,
  65        .final                  = sha256_final,
  66        .finup                  = sha256_finup,
  67        .descsize               = sizeof(struct sha256_state),
  68        .base.cra_name          = "sha256",
  69        .base.cra_driver_name   = "sha256-arm64",
  70        .base.cra_priority      = 100,
  71        .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
  72        .base.cra_blocksize     = SHA256_BLOCK_SIZE,
  73        .base.cra_module        = THIS_MODULE,
  74}, {
  75        .digestsize             = SHA224_DIGEST_SIZE,
  76        .init                   = sha224_base_init,
  77        .update                 = sha256_update,
  78        .final                  = sha256_final,
  79        .finup                  = sha256_finup,
  80        .descsize               = sizeof(struct sha256_state),
  81        .base.cra_name          = "sha224",
  82        .base.cra_driver_name   = "sha224-arm64",
  83        .base.cra_priority      = 100,
  84        .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
  85        .base.cra_blocksize     = SHA224_BLOCK_SIZE,
  86        .base.cra_module        = THIS_MODULE,
  87} };
  88
  89static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
  90                              unsigned int len)
  91{
  92        /*
  93         * Stacking and unstacking a substantial slice of the NEON register
  94         * file may significantly affect performance for small updates when
  95         * executing in interrupt context, so fall back to the scalar code
  96         * in that case.
  97         */
  98        if (!may_use_simd())
  99                return sha256_base_do_update(desc, data, len,
 100                                (sha256_block_fn *)sha256_block_data_order);
 101
 102        kernel_neon_begin();
 103        sha256_base_do_update(desc, data, len,
 104                                (sha256_block_fn *)sha256_block_neon);
 105        kernel_neon_end();
 106
 107        return 0;
 108}
 109
 110static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
 111                             unsigned int len, u8 *out)
 112{
 113        if (!may_use_simd()) {
 114                if (len)
 115                        sha256_base_do_update(desc, data, len,
 116                                (sha256_block_fn *)sha256_block_data_order);
 117                sha256_base_do_finalize(desc,
 118                                (sha256_block_fn *)sha256_block_data_order);
 119        } else {
 120                kernel_neon_begin();
 121                if (len)
 122                        sha256_base_do_update(desc, data, len,
 123                                (sha256_block_fn *)sha256_block_neon);
 124                sha256_base_do_finalize(desc,
 125                                (sha256_block_fn *)sha256_block_neon);
 126                kernel_neon_end();
 127        }
 128        return sha256_base_finish(desc, out);
 129}
 130
 131static int sha256_final_neon(struct shash_desc *desc, u8 *out)
 132{
 133        return sha256_finup_neon(desc, NULL, 0, out);
 134}
 135
 136static struct shash_alg neon_algs[] = { {
 137        .digestsize             = SHA256_DIGEST_SIZE,
 138        .init                   = sha256_base_init,
 139        .update                 = sha256_update_neon,
 140        .final                  = sha256_final_neon,
 141        .finup                  = sha256_finup_neon,
 142        .descsize               = sizeof(struct sha256_state),
 143        .base.cra_name          = "sha256",
 144        .base.cra_driver_name   = "sha256-arm64-neon",
 145        .base.cra_priority      = 150,
 146        .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
 147        .base.cra_blocksize     = SHA256_BLOCK_SIZE,
 148        .base.cra_module        = THIS_MODULE,
 149}, {
 150        .digestsize             = SHA224_DIGEST_SIZE,
 151        .init                   = sha224_base_init,
 152        .update                 = sha256_update_neon,
 153        .final                  = sha256_final_neon,
 154        .finup                  = sha256_finup_neon,
 155        .descsize               = sizeof(struct sha256_state),
 156        .base.cra_name          = "sha224",
 157        .base.cra_driver_name   = "sha224-arm64-neon",
 158        .base.cra_priority      = 150,
 159        .base.cra_flags         = CRYPTO_ALG_TYPE_SHASH,
 160        .base.cra_blocksize     = SHA224_BLOCK_SIZE,
 161        .base.cra_module        = THIS_MODULE,
 162} };
 163
 164static int __init sha256_mod_init(void)
 165{
 166        int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
 167        if (ret)
 168                return ret;
 169
 170        if (elf_hwcap & HWCAP_ASIMD) {
 171                ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
 172                if (ret)
 173                        crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 174        }
 175        return ret;
 176}
 177
 178static void __exit sha256_mod_fini(void)
 179{
 180        if (elf_hwcap & HWCAP_ASIMD)
 181                crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
 182        crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 183}
 184
 185module_init(sha256_mod_init);
 186module_exit(sha256_mod_fini);
 187