LXR linux/arch/x86/crypto/sha512_ssse3

   1/*
   2 * Cryptographic API.
   3 *
   4 * Glue code for the SHA512 Secure Hash Algorithm assembler
   5 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
   6 *
   7 * This file is based on sha512_generic.c
   8 *
   9 * Copyright (C) 2013 Intel Corporation
  10 * Author: Tim Chen <tim.c.chen@linux.intel.com>
  11 *
  12 * This program is free software; you can redistribute it and/or modify it
  13 * under the terms of the GNU General Public License as published by the Free
  14 * Software Foundation; either version 2 of the License, or (at your option)
  15 * any later version.
  16 *
  17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  21 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  22 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  23 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 * SOFTWARE.
  25 *
  26 */
  27
  28#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  29
  30#include <crypto/internal/hash.h>
  31#include <crypto/internal/simd.h>
  32#include <linux/init.h>
  33#include <linux/module.h>
  34#include <linux/mm.h>
  35#include <linux/string.h>
  36#include <linux/types.h>
  37#include <crypto/sha2.h>
  38#include <crypto/sha512_base.h>
  39#include <asm/simd.h>
  40
  41asmlinkage void sha512_transform_ssse3(struct sha512_state *state,
  42                                       const u8 *data, int blocks);
  43
  44static int sha512_update(struct shash_desc *desc, const u8 *data,
  45                       unsigned int len, sha512_block_fn *sha512_xform)
  46{
  47        struct sha512_state *sctx = shash_desc_ctx(desc);
  48
  49        if (!crypto_simd_usable() ||
  50            (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
  51                return crypto_sha512_update(desc, data, len);
  52
  53        /*
  54         * Make sure struct sha512_state begins directly with the SHA512
  55         * 512-bit internal state, as this is what the asm functions expect.
  56         */
  57        BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
  58
  59        kernel_fpu_begin();
  60        sha512_base_do_update(desc, data, len, sha512_xform);
  61        kernel_fpu_end();
  62
  63        return 0;
  64}
  65
  66static int sha512_finup(struct shash_desc *desc, const u8 *data,
  67              unsigned int len, u8 *out, sha512_block_fn *sha512_xform)
  68{
  69        if (!crypto_simd_usable())
  70                return crypto_sha512_finup(desc, data, len, out);
  71
  72        kernel_fpu_begin();
  73        if (len)
  74                sha512_base_do_update(desc, data, len, sha512_xform);
  75        sha512_base_do_finalize(desc, sha512_xform);
  76        kernel_fpu_end();
  77
  78        return sha512_base_finish(desc, out);
  79}
  80
  81static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
  82                       unsigned int len)
  83{
  84        return sha512_update(desc, data, len, sha512_transform_ssse3);
  85}
  86
  87static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
  88              unsigned int len, u8 *out)
  89{
  90        return sha512_finup(desc, data, len, out, sha512_transform_ssse3);
  91}
  92
  93/* Add padding and return the message digest. */
  94static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
  95{
  96        return sha512_ssse3_finup(desc, NULL, 0, out);
  97}
  98
  99static struct shash_alg sha512_ssse3_algs[] = { {
 100        .digestsize     =       SHA512_DIGEST_SIZE,
 101        .init           =       sha512_base_init,
 102        .update         =       sha512_ssse3_update,
 103        .final          =       sha512_ssse3_final,
 104        .finup          =       sha512_ssse3_finup,
 105        .descsize       =       sizeof(struct sha512_state),
 106        .base           =       {
 107                .cra_name       =       "sha512",
 108                .cra_driver_name =      "sha512-ssse3",
 109                .cra_priority   =       150,
 110                .cra_blocksize  =       SHA512_BLOCK_SIZE,
 111                .cra_module     =       THIS_MODULE,
 112        }
 113},  {
 114        .digestsize     =       SHA384_DIGEST_SIZE,
 115        .init           =       sha384_base_init,
 116        .update         =       sha512_ssse3_update,
 117        .final          =       sha512_ssse3_final,
 118        .finup          =       sha512_ssse3_finup,
 119        .descsize       =       sizeof(struct sha512_state),
 120        .base           =       {
 121                .cra_name       =       "sha384",
 122                .cra_driver_name =      "sha384-ssse3",
 123                .cra_priority   =       150,
 124                .cra_blocksize  =       SHA384_BLOCK_SIZE,
 125                .cra_module     =       THIS_MODULE,
 126        }
 127} };
 128
 129static int register_sha512_ssse3(void)
 130{
 131        if (boot_cpu_has(X86_FEATURE_SSSE3))
 132                return crypto_register_shashes(sha512_ssse3_algs,
 133                        ARRAY_SIZE(sha512_ssse3_algs));
 134        return 0;
 135}
 136
 137static void unregister_sha512_ssse3(void)
 138{
 139        if (boot_cpu_has(X86_FEATURE_SSSE3))
 140                crypto_unregister_shashes(sha512_ssse3_algs,
 141                        ARRAY_SIZE(sha512_ssse3_algs));
 142}
 143
 144asmlinkage void sha512_transform_avx(struct sha512_state *state,
 145                                     const u8 *data, int blocks);
 146static bool avx_usable(void)
 147{
 148        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
 149                if (boot_cpu_has(X86_FEATURE_AVX))
 150                        pr_info("AVX detected but unusable.\n");
 151                return false;
 152        }
 153
 154        return true;
 155}
 156
 157static int sha512_avx_update(struct shash_desc *desc, const u8 *data,
 158                       unsigned int len)
 159{
 160        return sha512_update(desc, data, len, sha512_transform_avx);
 161}
 162
 163static int sha512_avx_finup(struct shash_desc *desc, const u8 *data,
 164              unsigned int len, u8 *out)
 165{
 166        return sha512_finup(desc, data, len, out, sha512_transform_avx);
 167}
 168
 169/* Add padding and return the message digest. */
 170static int sha512_avx_final(struct shash_desc *desc, u8 *out)
 171{
 172        return sha512_avx_finup(desc, NULL, 0, out);
 173}
 174
 175static struct shash_alg sha512_avx_algs[] = { {
 176        .digestsize     =       SHA512_DIGEST_SIZE,
 177        .init           =       sha512_base_init,
 178        .update         =       sha512_avx_update,
 179        .final          =       sha512_avx_final,
 180        .finup          =       sha512_avx_finup,
 181        .descsize       =       sizeof(struct sha512_state),
 182        .base           =       {
 183                .cra_name       =       "sha512",
 184                .cra_driver_name =      "sha512-avx",
 185                .cra_priority   =       160,
 186                .cra_blocksize  =       SHA512_BLOCK_SIZE,
 187                .cra_module     =       THIS_MODULE,
 188        }
 189},  {
 190        .digestsize     =       SHA384_DIGEST_SIZE,
 191        .init           =       sha384_base_init,
 192        .update         =       sha512_avx_update,
 193        .final          =       sha512_avx_final,
 194        .finup          =       sha512_avx_finup,
 195        .descsize       =       sizeof(struct sha512_state),
 196        .base           =       {
 197                .cra_name       =       "sha384",
 198                .cra_driver_name =      "sha384-avx",
 199                .cra_priority   =       160,
 200                .cra_blocksize  =       SHA384_BLOCK_SIZE,
 201                .cra_module     =       THIS_MODULE,
 202        }
 203} };
 204
 205static int register_sha512_avx(void)
 206{
 207        if (avx_usable())
 208                return crypto_register_shashes(sha512_avx_algs,
 209                        ARRAY_SIZE(sha512_avx_algs));
 210        return 0;
 211}
 212
 213static void unregister_sha512_avx(void)
 214{
 215        if (avx_usable())
 216                crypto_unregister_shashes(sha512_avx_algs,
 217                        ARRAY_SIZE(sha512_avx_algs));
 218}
 219
 220asmlinkage void sha512_transform_rorx(struct sha512_state *state,
 221                                      const u8 *data, int blocks);
 222
 223static int sha512_avx2_update(struct shash_desc *desc, const u8 *data,
 224                       unsigned int len)
 225{
 226        return sha512_update(desc, data, len, sha512_transform_rorx);
 227}
 228
 229static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data,
 230              unsigned int len, u8 *out)
 231{
 232        return sha512_finup(desc, data, len, out, sha512_transform_rorx);
 233}
 234
 235/* Add padding and return the message digest. */
 236static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
 237{
 238        return sha512_avx2_finup(desc, NULL, 0, out);
 239}
 240
 241static struct shash_alg sha512_avx2_algs[] = { {
 242        .digestsize     =       SHA512_DIGEST_SIZE,
 243        .init           =       sha512_base_init,
 244        .update         =       sha512_avx2_update,
 245        .final          =       sha512_avx2_final,
 246        .finup          =       sha512_avx2_finup,
 247        .descsize       =       sizeof(struct sha512_state),
 248        .base           =       {
 249                .cra_name       =       "sha512",
 250                .cra_driver_name =      "sha512-avx2",
 251                .cra_priority   =       170,
 252                .cra_blocksize  =       SHA512_BLOCK_SIZE,
 253                .cra_module     =       THIS_MODULE,
 254        }
 255},  {
 256        .digestsize     =       SHA384_DIGEST_SIZE,
 257        .init           =       sha384_base_init,
 258        .update         =       sha512_avx2_update,
 259        .final          =       sha512_avx2_final,
 260        .finup          =       sha512_avx2_finup,
 261        .descsize       =       sizeof(struct sha512_state),
 262        .base           =       {
 263                .cra_name       =       "sha384",
 264                .cra_driver_name =      "sha384-avx2",
 265                .cra_priority   =       170,
 266                .cra_blocksize  =       SHA384_BLOCK_SIZE,
 267                .cra_module     =       THIS_MODULE,
 268        }
 269} };
 270
 271static bool avx2_usable(void)
 272{
 273        if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
 274                    boot_cpu_has(X86_FEATURE_BMI2))
 275                return true;
 276
 277        return false;
 278}
 279
 280static int register_sha512_avx2(void)
 281{
 282        if (avx2_usable())
 283                return crypto_register_shashes(sha512_avx2_algs,
 284                        ARRAY_SIZE(sha512_avx2_algs));
 285        return 0;
 286}
 287
 288static void unregister_sha512_avx2(void)
 289{
 290        if (avx2_usable())
 291                crypto_unregister_shashes(sha512_avx2_algs,
 292                        ARRAY_SIZE(sha512_avx2_algs));
 293}
 294
 295static int __init sha512_ssse3_mod_init(void)
 296{
 297
 298        if (register_sha512_ssse3())
 299                goto fail;
 300
 301        if (register_sha512_avx()) {
 302                unregister_sha512_ssse3();
 303                goto fail;
 304        }
 305
 306        if (register_sha512_avx2()) {
 307                unregister_sha512_avx();
 308                unregister_sha512_ssse3();
 309                goto fail;
 310        }
 311
 312        return 0;
 313fail:
 314        return -ENODEV;
 315}
 316
 317static void __exit sha512_ssse3_mod_fini(void)
 318{
 319        unregister_sha512_avx2();
 320        unregister_sha512_avx();
 321        unregister_sha512_ssse3();
 322}
 323
 324module_init(sha512_ssse3_mod_init);
 325module_exit(sha512_ssse3_mod_fini);
 326
 327MODULE_LICENSE("GPL");
 328MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 329
 330MODULE_ALIAS_CRYPTO("sha512");
 331MODULE_ALIAS_CRYPTO("sha512-ssse3");
 332MODULE_ALIAS_CRYPTO("sha512-avx");
 333MODULE_ALIAS_CRYPTO("sha512-avx2");
 334MODULE_ALIAS_CRYPTO("sha384");
 335MODULE_ALIAS_CRYPTO("sha384-ssse3");
 336MODULE_ALIAS_CRYPTO("sha384-avx");
 337MODULE_ALIAS_CRYPTO("sha384-avx2");
 338