linux/arch/sparc/crypto/des_glue.c
<<
>>
Prefs
   1/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
   2 *
   3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
   4 */
   5
   6#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
   7
   8#include <linux/crypto.h>
   9#include <linux/init.h>
  10#include <linux/module.h>
  11#include <linux/mm.h>
  12#include <linux/types.h>
  13#include <crypto/algapi.h>
  14#include <crypto/des.h>
  15
  16#include <asm/fpumacro.h>
  17#include <asm/pstate.h>
  18#include <asm/elf.h>
  19
  20#include "opcodes.h"
  21
  22struct des_sparc64_ctx {
  23        u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
  24        u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
  25};
  26
  27struct des3_ede_sparc64_ctx {
  28        u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
  29        u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
  30};
  31
  32static void encrypt_to_decrypt(u64 *d, const u64 *e)
  33{
  34        const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
  35        int i;
  36
  37        for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
  38                *d++ = *s--;
  39}
  40
  41extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
  42
  43static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
  44                       unsigned int keylen)
  45{
  46        struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
  47        u32 *flags = &tfm->crt_flags;
  48        u32 tmp[DES_EXPKEY_WORDS];
  49        int ret;
  50
  51        /* Even though we have special instructions for key expansion,
  52         * we call des_ekey() so that we don't have to write our own
  53         * weak key detection code.
  54         */
  55        ret = des_ekey(tmp, key);
  56        if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
  57                *flags |= CRYPTO_TFM_RES_WEAK_KEY;
  58                return -EINVAL;
  59        }
  60
  61        des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
  62        encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
  63
  64        return 0;
  65}
  66
  67extern void des_sparc64_crypt(const u64 *key, const u64 *input,
  68                              u64 *output);
  69
  70static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  71{
  72        struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
  73        const u64 *K = ctx->encrypt_expkey;
  74
  75        des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
  76}
  77
  78static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  79{
  80        struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
  81        const u64 *K = ctx->decrypt_expkey;
  82
  83        des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
  84}
  85
  86extern void des_sparc64_load_keys(const u64 *key);
  87
  88extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
  89                                  unsigned int len);
  90
  91#define DES_BLOCK_MASK  (~(DES_BLOCK_SIZE - 1))
  92
  93static int __ecb_crypt(struct blkcipher_desc *desc,
  94                       struct scatterlist *dst, struct scatterlist *src,
  95                       unsigned int nbytes, bool encrypt)
  96{
  97        struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  98        struct blkcipher_walk walk;
  99        int err;
 100
 101        blkcipher_walk_init(&walk, dst, src, nbytes);
 102        err = blkcipher_walk_virt(desc, &walk);
 103        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 104
 105        if (encrypt)
 106                des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
 107        else
 108                des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
 109        while ((nbytes = walk.nbytes)) {
 110                unsigned int block_len = nbytes & DES_BLOCK_MASK;
 111
 112                if (likely(block_len)) {
 113                        des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
 114                                              (u64 *) walk.dst.virt.addr,
 115                                              block_len);
 116                }
 117                nbytes &= DES_BLOCK_SIZE - 1;
 118                err = blkcipher_walk_done(desc, &walk, nbytes);
 119        }
 120        fprs_write(0);
 121        return err;
 122}
 123
 124static int ecb_encrypt(struct blkcipher_desc *desc,
 125                       struct scatterlist *dst, struct scatterlist *src,
 126                       unsigned int nbytes)
 127{
 128        return __ecb_crypt(desc, dst, src, nbytes, true);
 129}
 130
 131static int ecb_decrypt(struct blkcipher_desc *desc,
 132                       struct scatterlist *dst, struct scatterlist *src,
 133                       unsigned int nbytes)
 134{
 135        return __ecb_crypt(desc, dst, src, nbytes, false);
 136}
 137
 138extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
 139                                    unsigned int len, u64 *iv);
 140
 141static int cbc_encrypt(struct blkcipher_desc *desc,
 142                       struct scatterlist *dst, struct scatterlist *src,
 143                       unsigned int nbytes)
 144{
 145        struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 146        struct blkcipher_walk walk;
 147        int err;
 148
 149        blkcipher_walk_init(&walk, dst, src, nbytes);
 150        err = blkcipher_walk_virt(desc, &walk);
 151        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 152
 153        des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
 154        while ((nbytes = walk.nbytes)) {
 155                unsigned int block_len = nbytes & DES_BLOCK_MASK;
 156
 157                if (likely(block_len)) {
 158                        des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
 159                                                (u64 *) walk.dst.virt.addr,
 160                                                block_len, (u64 *) walk.iv);
 161                }
 162                nbytes &= DES_BLOCK_SIZE - 1;
 163                err = blkcipher_walk_done(desc, &walk, nbytes);
 164        }
 165        fprs_write(0);
 166        return err;
 167}
 168
 169extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
 170                                    unsigned int len, u64 *iv);
 171
 172static int cbc_decrypt(struct blkcipher_desc *desc,
 173                       struct scatterlist *dst, struct scatterlist *src,
 174                       unsigned int nbytes)
 175{
 176        struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 177        struct blkcipher_walk walk;
 178        int err;
 179
 180        blkcipher_walk_init(&walk, dst, src, nbytes);
 181        err = blkcipher_walk_virt(desc, &walk);
 182        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 183
 184        des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
 185        while ((nbytes = walk.nbytes)) {
 186                unsigned int block_len = nbytes & DES_BLOCK_MASK;
 187
 188                if (likely(block_len)) {
 189                        des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
 190                                                (u64 *) walk.dst.virt.addr,
 191                                                block_len, (u64 *) walk.iv);
 192                }
 193                nbytes &= DES_BLOCK_SIZE - 1;
 194                err = blkcipher_walk_done(desc, &walk, nbytes);
 195        }
 196        fprs_write(0);
 197        return err;
 198}
 199
 200static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
 201                            unsigned int keylen)
 202{
 203        struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
 204        const u32 *K = (const u32 *)key;
 205        u32 *flags = &tfm->crt_flags;
 206        u64 k1[DES_EXPKEY_WORDS / 2];
 207        u64 k2[DES_EXPKEY_WORDS / 2];
 208        u64 k3[DES_EXPKEY_WORDS / 2];
 209
 210        if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
 211                     !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
 212                     (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 213                *flags |= CRYPTO_TFM_RES_WEAK_KEY;
 214                return -EINVAL;
 215        }
 216
 217        des_sparc64_key_expand((const u32 *)key, k1);
 218        key += DES_KEY_SIZE;
 219        des_sparc64_key_expand((const u32 *)key, k2);
 220        key += DES_KEY_SIZE;
 221        des_sparc64_key_expand((const u32 *)key, k3);
 222
 223        memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
 224        encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
 225        memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
 226               &k3[0], sizeof(k3));
 227
 228        encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
 229        memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
 230               &k2[0], sizeof(k2));
 231        encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
 232                           &k1[0]);
 233
 234        return 0;
 235}
 236
 237extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
 238                                   u64 *output);
 239
 240static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 241{
 242        struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
 243        const u64 *K = ctx->encrypt_expkey;
 244
 245        des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
 246}
 247
 248static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 249{
 250        struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
 251        const u64 *K = ctx->decrypt_expkey;
 252
 253        des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
 254}
 255
 256extern void des3_ede_sparc64_load_keys(const u64 *key);
 257
 258extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
 259                                       u64 *output, unsigned int len);
 260
 261static int __ecb3_crypt(struct blkcipher_desc *desc,
 262                        struct scatterlist *dst, struct scatterlist *src,
 263                        unsigned int nbytes, bool encrypt)
 264{
 265        struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 266        struct blkcipher_walk walk;
 267        const u64 *K;
 268        int err;
 269
 270        blkcipher_walk_init(&walk, dst, src, nbytes);
 271        err = blkcipher_walk_virt(desc, &walk);
 272        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 273
 274        if (encrypt)
 275                K = &ctx->encrypt_expkey[0];
 276        else
 277                K = &ctx->decrypt_expkey[0];
 278        des3_ede_sparc64_load_keys(K);
 279        while ((nbytes = walk.nbytes)) {
 280                unsigned int block_len = nbytes & DES_BLOCK_MASK;
 281
 282                if (likely(block_len)) {
 283                        const u64 *src64 = (const u64 *)walk.src.virt.addr;
 284                        des3_ede_sparc64_ecb_crypt(K, src64,
 285                                                   (u64 *) walk.dst.virt.addr,
 286                                                   block_len);
 287                }
 288                nbytes &= DES_BLOCK_SIZE - 1;
 289                err = blkcipher_walk_done(desc, &walk, nbytes);
 290        }
 291        fprs_write(0);
 292        return err;
 293}
 294
 295static int ecb3_encrypt(struct blkcipher_desc *desc,
 296                       struct scatterlist *dst, struct scatterlist *src,
 297                       unsigned int nbytes)
 298{
 299        return __ecb3_crypt(desc, dst, src, nbytes, true);
 300}
 301
 302static int ecb3_decrypt(struct blkcipher_desc *desc,
 303                       struct scatterlist *dst, struct scatterlist *src,
 304                       unsigned int nbytes)
 305{
 306        return __ecb3_crypt(desc, dst, src, nbytes, false);
 307}
 308
 309extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
 310                                         u64 *output, unsigned int len,
 311                                         u64 *iv);
 312
 313static int cbc3_encrypt(struct blkcipher_desc *desc,
 314                        struct scatterlist *dst, struct scatterlist *src,
 315                        unsigned int nbytes)
 316{
 317        struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 318        struct blkcipher_walk walk;
 319        const u64 *K;
 320        int err;
 321
 322        blkcipher_walk_init(&walk, dst, src, nbytes);
 323        err = blkcipher_walk_virt(desc, &walk);
 324        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 325
 326        K = &ctx->encrypt_expkey[0];
 327        des3_ede_sparc64_load_keys(K);
 328        while ((nbytes = walk.nbytes)) {
 329                unsigned int block_len = nbytes & DES_BLOCK_MASK;
 330
 331                if (likely(block_len)) {
 332                        const u64 *src64 = (const u64 *)walk.src.virt.addr;
 333                        des3_ede_sparc64_cbc_encrypt(K, src64,
 334                                                     (u64 *) walk.dst.virt.addr,
 335                                                     block_len,
 336                                                     (u64 *) walk.iv);
 337                }
 338                nbytes &= DES_BLOCK_SIZE - 1;
 339                err = blkcipher_walk_done(desc, &walk, nbytes);
 340        }
 341        fprs_write(0);
 342        return err;
 343}
 344
 345extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
 346                                         u64 *output, unsigned int len,
 347                                         u64 *iv);
 348
 349static int cbc3_decrypt(struct blkcipher_desc *desc,
 350                        struct scatterlist *dst, struct scatterlist *src,
 351                        unsigned int nbytes)
 352{
 353        struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 354        struct blkcipher_walk walk;
 355        const u64 *K;
 356        int err;
 357
 358        blkcipher_walk_init(&walk, dst, src, nbytes);
 359        err = blkcipher_walk_virt(desc, &walk);
 360        desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 361
 362        K = &ctx->decrypt_expkey[0];
 363        des3_ede_sparc64_load_keys(K);
 364        while ((nbytes = walk.nbytes)) {
 365                unsigned int block_len = nbytes & DES_BLOCK_MASK;
 366
 367                if (likely(block_len)) {
 368                        const u64 *src64 = (const u64 *)walk.src.virt.addr;
 369                        des3_ede_sparc64_cbc_decrypt(K, src64,
 370                                                     (u64 *) walk.dst.virt.addr,
 371                                                     block_len,
 372                                                     (u64 *) walk.iv);
 373                }
 374                nbytes &= DES_BLOCK_SIZE - 1;
 375                err = blkcipher_walk_done(desc, &walk, nbytes);
 376        }
 377        fprs_write(0);
 378        return err;
 379}
 380
 381static struct crypto_alg algs[] = { {
 382        .cra_name               = "des",
 383        .cra_driver_name        = "des-sparc64",
 384        .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 385        .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 386        .cra_blocksize          = DES_BLOCK_SIZE,
 387        .cra_ctxsize            = sizeof(struct des_sparc64_ctx),
 388        .cra_alignmask          = 7,
 389        .cra_module             = THIS_MODULE,
 390        .cra_u  = {
 391                .cipher = {
 392                        .cia_min_keysize        = DES_KEY_SIZE,
 393                        .cia_max_keysize        = DES_KEY_SIZE,
 394                        .cia_setkey             = des_set_key,
 395                        .cia_encrypt            = des_encrypt,
 396                        .cia_decrypt            = des_decrypt
 397                }
 398        }
 399}, {
 400        .cra_name               = "ecb(des)",
 401        .cra_driver_name        = "ecb-des-sparc64",
 402        .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 403        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 404        .cra_blocksize          = DES_BLOCK_SIZE,
 405        .cra_ctxsize            = sizeof(struct des_sparc64_ctx),
 406        .cra_alignmask          = 7,
 407        .cra_type               = &crypto_blkcipher_type,
 408        .cra_module             = THIS_MODULE,
 409        .cra_u = {
 410                .blkcipher = {
 411                        .min_keysize    = DES_KEY_SIZE,
 412                        .max_keysize    = DES_KEY_SIZE,
 413                        .setkey         = des_set_key,
 414                        .encrypt        = ecb_encrypt,
 415                        .decrypt        = ecb_decrypt,
 416                },
 417        },
 418}, {
 419        .cra_name               = "cbc(des)",
 420        .cra_driver_name        = "cbc-des-sparc64",
 421        .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 422        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 423        .cra_blocksize          = DES_BLOCK_SIZE,
 424        .cra_ctxsize            = sizeof(struct des_sparc64_ctx),
 425        .cra_alignmask          = 7,
 426        .cra_type               = &crypto_blkcipher_type,
 427        .cra_module             = THIS_MODULE,
 428        .cra_u = {
 429                .blkcipher = {
 430                        .min_keysize    = DES_KEY_SIZE,
 431                        .max_keysize    = DES_KEY_SIZE,
 432                        .ivsize         = DES_BLOCK_SIZE,
 433                        .setkey         = des_set_key,
 434                        .encrypt        = cbc_encrypt,
 435                        .decrypt        = cbc_decrypt,
 436                },
 437        },
 438}, {
 439        .cra_name               = "des3_ede",
 440        .cra_driver_name        = "des3_ede-sparc64",
 441        .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 442        .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 443        .cra_blocksize          = DES3_EDE_BLOCK_SIZE,
 444        .cra_ctxsize            = sizeof(struct des3_ede_sparc64_ctx),
 445        .cra_alignmask          = 7,
 446        .cra_module             = THIS_MODULE,
 447        .cra_u  = {
 448                .cipher = {
 449                        .cia_min_keysize        = DES3_EDE_KEY_SIZE,
 450                        .cia_max_keysize        = DES3_EDE_KEY_SIZE,
 451                        .cia_setkey             = des3_ede_set_key,
 452                        .cia_encrypt            = des3_ede_encrypt,
 453                        .cia_decrypt            = des3_ede_decrypt
 454                }
 455        }
 456}, {
 457        .cra_name               = "ecb(des3_ede)",
 458        .cra_driver_name        = "ecb-des3_ede-sparc64",
 459        .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 460        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 461        .cra_blocksize          = DES3_EDE_BLOCK_SIZE,
 462        .cra_ctxsize            = sizeof(struct des3_ede_sparc64_ctx),
 463        .cra_alignmask          = 7,
 464        .cra_type               = &crypto_blkcipher_type,
 465        .cra_module             = THIS_MODULE,
 466        .cra_u = {
 467                .blkcipher = {
 468                        .min_keysize    = DES3_EDE_KEY_SIZE,
 469                        .max_keysize    = DES3_EDE_KEY_SIZE,
 470                        .setkey         = des3_ede_set_key,
 471                        .encrypt        = ecb3_encrypt,
 472                        .decrypt        = ecb3_decrypt,
 473                },
 474        },
 475}, {
 476        .cra_name               = "cbc(des3_ede)",
 477        .cra_driver_name        = "cbc-des3_ede-sparc64",
 478        .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
 479        .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 480        .cra_blocksize          = DES3_EDE_BLOCK_SIZE,
 481        .cra_ctxsize            = sizeof(struct des3_ede_sparc64_ctx),
 482        .cra_alignmask          = 7,
 483        .cra_type               = &crypto_blkcipher_type,
 484        .cra_module             = THIS_MODULE,
 485        .cra_u = {
 486                .blkcipher = {
 487                        .min_keysize    = DES3_EDE_KEY_SIZE,
 488                        .max_keysize    = DES3_EDE_KEY_SIZE,
 489                        .ivsize         = DES3_EDE_BLOCK_SIZE,
 490                        .setkey         = des3_ede_set_key,
 491                        .encrypt        = cbc3_encrypt,
 492                        .decrypt        = cbc3_decrypt,
 493                },
 494        },
 495} };
 496
 497static bool __init sparc64_has_des_opcode(void)
 498{
 499        unsigned long cfr;
 500
 501        if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
 502                return false;
 503
 504        __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
 505        if (!(cfr & CFR_DES))
 506                return false;
 507
 508        return true;
 509}
 510
 511static int __init des_sparc64_mod_init(void)
 512{
 513        int i;
 514
 515        for (i = 0; i < ARRAY_SIZE(algs); i++)
 516                INIT_LIST_HEAD(&algs[i].cra_list);
 517
 518        if (sparc64_has_des_opcode()) {
 519                pr_info("Using sparc64 des opcodes optimized DES implementation\n");
 520                return crypto_register_algs(algs, ARRAY_SIZE(algs));
 521        }
 522        pr_info("sparc64 des opcodes not available.\n");
 523        return -ENODEV;
 524}
 525
 526static void __exit des_sparc64_mod_fini(void)
 527{
 528        crypto_unregister_algs(algs, ARRAY_SIZE(algs));
 529}
 530
 531module_init(des_sparc64_mod_init);
 532module_exit(des_sparc64_mod_fini);
 533
 534MODULE_LICENSE("GPL");
 535MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
 536
 537MODULE_ALIAS_CRYPTO("des");
 538MODULE_ALIAS_CRYPTO("des3_ede");
 539
 540#include "crop_devid.c"
 541