LXR linux/arch/powerpc/crypto/crct10dif-vpmsum

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Calculate a CRC T10-DIF with vpmsum acceleration
   4 *
   5 * Copyright 2017, Daniel Axtens, IBM Corporation.
   6 * [based on crc32c-vpmsum_glue.c]
   7 */
   8
   9#include <linux/crc-t10dif.h>
  10#include <crypto/internal/hash.h>
  11#include <crypto/internal/simd.h>
  12#include <linux/init.h>
  13#include <linux/module.h>
  14#include <linux/string.h>
  15#include <linux/kernel.h>
  16#include <linux/cpufeature.h>
  17#include <asm/simd.h>
  18#include <asm/switch_to.h>
  19
  20#define VMX_ALIGN               16
  21#define VMX_ALIGN_MASK          (VMX_ALIGN-1)
  22
  23#define VECTOR_BREAKPOINT       64
  24
  25u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
  26
  27static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len)
  28{
  29        unsigned int prealign;
  30        unsigned int tail;
  31        u32 crc = crci;
  32
  33        if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
  34                return crc_t10dif_generic(crc, p, len);
  35
  36        if ((unsigned long)p & VMX_ALIGN_MASK) {
  37                prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
  38                crc = crc_t10dif_generic(crc, p, prealign);
  39                len -= prealign;
  40                p += prealign;
  41        }
  42
  43        if (len & ~VMX_ALIGN_MASK) {
  44                crc <<= 16;
  45                preempt_disable();
  46                pagefault_disable();
  47                enable_kernel_altivec();
  48                crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
  49                disable_kernel_altivec();
  50                pagefault_enable();
  51                preempt_enable();
  52                crc >>= 16;
  53        }
  54
  55        tail = len & VMX_ALIGN_MASK;
  56        if (tail) {
  57                p += len & ~VMX_ALIGN_MASK;
  58                crc = crc_t10dif_generic(crc, p, tail);
  59        }
  60
  61        return crc & 0xffff;
  62}
  63
  64static int crct10dif_vpmsum_init(struct shash_desc *desc)
  65{
  66        u16 *crc = shash_desc_ctx(desc);
  67
  68        *crc = 0;
  69        return 0;
  70}
  71
  72static int crct10dif_vpmsum_update(struct shash_desc *desc, const u8 *data,
  73                            unsigned int length)
  74{
  75        u16 *crc = shash_desc_ctx(desc);
  76
  77        *crc = crct10dif_vpmsum(*crc, data, length);
  78
  79        return 0;
  80}
  81
  82
  83static int crct10dif_vpmsum_final(struct shash_desc *desc, u8 *out)
  84{
  85        u16 *crcp = shash_desc_ctx(desc);
  86
  87        *(u16 *)out = *crcp;
  88        return 0;
  89}
  90
  91static struct shash_alg alg = {
  92        .init           = crct10dif_vpmsum_init,
  93        .update         = crct10dif_vpmsum_update,
  94        .final          = crct10dif_vpmsum_final,
  95        .descsize       = CRC_T10DIF_DIGEST_SIZE,
  96        .digestsize     = CRC_T10DIF_DIGEST_SIZE,
  97        .base           = {
  98                .cra_name               = "crct10dif",
  99                .cra_driver_name        = "crct10dif-vpmsum",
 100                .cra_priority           = 200,
 101                .cra_blocksize          = CRC_T10DIF_BLOCK_SIZE,
 102                .cra_module             = THIS_MODULE,
 103        }
 104};
 105
 106static int __init crct10dif_vpmsum_mod_init(void)
 107{
 108        if (!cpu_has_feature(CPU_FTR_ARCH_207S))
 109                return -ENODEV;
 110
 111        return crypto_register_shash(&alg);
 112}
 113
 114static void __exit crct10dif_vpmsum_mod_fini(void)
 115{
 116        crypto_unregister_shash(&alg);
 117}
 118
 119module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crct10dif_vpmsum_mod_init);
 120module_exit(crct10dif_vpmsum_mod_fini);
 121
 122MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>");
 123MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions");
 124MODULE_LICENSE("GPL");
 125MODULE_ALIAS_CRYPTO("crct10dif");
 126MODULE_ALIAS_CRYPTO("crct10dif-vpmsum");
 127