linux/drivers/crypto/caam/caamalg.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * caam - Freescale FSL CAAM support for crypto API
   4 *
   5 * Copyright 2008-2011 Freescale Semiconductor, Inc.
   6 * Copyright 2016-2019 NXP
   7 *
   8 * Based on talitos crypto API driver.
   9 *
  10 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
  11 *
  12 * ---------------                     ---------------
  13 * | JobDesc #1  |-------------------->|  ShareDesc  |
  14 * | *(packet 1) |                     |   (PDB)     |
  15 * ---------------      |------------->|  (hashKey)  |
  16 *       .              |              | (cipherKey) |
  17 *       .              |    |-------->| (operation) |
  18 * ---------------      |    |         ---------------
  19 * | JobDesc #2  |------|    |
  20 * | *(packet 2) |           |
  21 * ---------------           |
  22 *       .                   |
  23 *       .                   |
  24 * ---------------           |
  25 * | JobDesc #3  |------------
  26 * | *(packet 3) |
  27 * ---------------
  28 *
  29 * The SharedDesc never changes for a connection unless rekeyed, but
  30 * each packet will likely be in a different place. So all we need
  31 * to know to process the packet is where the input is, where the
  32 * output goes, and what context we want to process with. Context is
  33 * in the SharedDesc, packet references in the JobDesc.
  34 *
  35 * So, a job desc looks like:
  36 *
  37 * ---------------------
  38 * | Header            |
  39 * | ShareDesc Pointer |
  40 * | SEQ_OUT_PTR       |
  41 * | (output buffer)   |
  42 * | (output length)   |
  43 * | SEQ_IN_PTR        |
  44 * | (input buffer)    |
  45 * | (input length)    |
  46 * ---------------------
  47 */
  48
  49#include "compat.h"
  50
  51#include "regs.h"
  52#include "intern.h"
  53#include "desc_constr.h"
  54#include "jr.h"
  55#include "error.h"
  56#include "sg_sw_sec4.h"
  57#include "key_gen.h"
  58#include "caamalg_desc.h"
  59#include <crypto/engine.h>
  60#include <crypto/xts.h>
  61#include <asm/unaligned.h>
  62
  63/*
  64 * crypto alg
  65 */
  66#define CAAM_CRA_PRIORITY               3000
  67/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  68#define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  69                                         CTR_RFC3686_NONCE_SIZE + \
  70                                         SHA512_DIGEST_SIZE * 2)
  71
  72#define AEAD_DESC_JOB_IO_LEN            (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
  73#define GCM_DESC_JOB_IO_LEN             (AEAD_DESC_JOB_IO_LEN + \
  74                                         CAAM_CMD_SZ * 4)
  75#define AUTHENC_DESC_JOB_IO_LEN         (AEAD_DESC_JOB_IO_LEN + \
  76                                         CAAM_CMD_SZ * 5)
  77
  78#define CHACHAPOLY_DESC_JOB_IO_LEN      (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
  79
  80#define DESC_MAX_USED_BYTES             (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
  81#define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  82
  83struct caam_alg_entry {
  84        int class1_alg_type;
  85        int class2_alg_type;
  86        bool rfc3686;
  87        bool geniv;
  88        bool nodkp;
  89};
  90
  91struct caam_aead_alg {
  92        struct aead_alg aead;
  93        struct caam_alg_entry caam;
  94        bool registered;
  95};
  96
  97struct caam_skcipher_alg {
  98        struct skcipher_alg skcipher;
  99        struct caam_alg_entry caam;
 100        bool registered;
 101};
 102
 103/*
 104 * per-session context
 105 */
 106struct caam_ctx {
 107        struct crypto_engine_ctx enginectx;
 108        u32 sh_desc_enc[DESC_MAX_USED_LEN];
 109        u32 sh_desc_dec[DESC_MAX_USED_LEN];
 110        u8 key[CAAM_MAX_KEY_SIZE];
 111        dma_addr_t sh_desc_enc_dma;
 112        dma_addr_t sh_desc_dec_dma;
 113        dma_addr_t key_dma;
 114        enum dma_data_direction dir;
 115        struct device *jrdev;
 116        struct alginfo adata;
 117        struct alginfo cdata;
 118        unsigned int authsize;
 119        bool xts_key_fallback;
 120        struct crypto_skcipher *fallback;
 121};
 122
 123struct caam_skcipher_req_ctx {
 124        struct skcipher_edesc *edesc;
 125        struct skcipher_request fallback_req;
 126};
 127
 128struct caam_aead_req_ctx {
 129        struct aead_edesc *edesc;
 130};
 131
 132static int aead_null_set_sh_desc(struct crypto_aead *aead)
 133{
 134        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 135        struct device *jrdev = ctx->jrdev;
 136        struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 137        u32 *desc;
 138        int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
 139                        ctx->adata.keylen_pad;
 140
 141        /*
 142         * Job Descriptor and Shared Descriptors
 143         * must all fit into the 64-word Descriptor h/w Buffer
 144         */
 145        if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
 146                ctx->adata.key_inline = true;
 147                ctx->adata.key_virt = ctx->key;
 148        } else {
 149                ctx->adata.key_inline = false;
 150                ctx->adata.key_dma = ctx->key_dma;
 151        }
 152
 153        /* aead_encrypt shared descriptor */
 154        desc = ctx->sh_desc_enc;
 155        cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
 156                                    ctrlpriv->era);
 157        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 158                                   desc_bytes(desc), ctx->dir);
 159
 160        /*
 161         * Job Descriptor and Shared Descriptors
 162         * must all fit into the 64-word Descriptor h/w Buffer
 163         */
 164        if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
 165                ctx->adata.key_inline = true;
 166                ctx->adata.key_virt = ctx->key;
 167        } else {
 168                ctx->adata.key_inline = false;
 169                ctx->adata.key_dma = ctx->key_dma;
 170        }
 171
 172        /* aead_decrypt shared descriptor */
 173        desc = ctx->sh_desc_dec;
 174        cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
 175                                    ctrlpriv->era);
 176        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 177                                   desc_bytes(desc), ctx->dir);
 178
 179        return 0;
 180}
 181
 182static int aead_set_sh_desc(struct crypto_aead *aead)
 183{
 184        struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 185                                                 struct caam_aead_alg, aead);
 186        unsigned int ivsize = crypto_aead_ivsize(aead);
 187        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 188        struct device *jrdev = ctx->jrdev;
 189        struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 190        u32 ctx1_iv_off = 0;
 191        u32 *desc, *nonce = NULL;
 192        u32 inl_mask;
 193        unsigned int data_len[2];
 194        const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 195                               OP_ALG_AAI_CTR_MOD128);
 196        const bool is_rfc3686 = alg->caam.rfc3686;
 197
 198        if (!ctx->authsize)
 199                return 0;
 200
 201        /* NULL encryption / decryption */
 202        if (!ctx->cdata.keylen)
 203                return aead_null_set_sh_desc(aead);
 204
 205        /*
 206         * AES-CTR needs to load IV in CONTEXT1 reg
 207         * at an offset of 128bits (16bytes)
 208         * CONTEXT1[255:128] = IV
 209         */
 210        if (ctr_mode)
 211                ctx1_iv_off = 16;
 212
 213        /*
 214         * RFC3686 specific:
 215         *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 216         */
 217        if (is_rfc3686) {
 218                ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 219                nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 220                                ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 221        }
 222
 223        /*
 224         * In case |user key| > |derived key|, using DKP<imm,imm>
 225         * would result in invalid opcodes (last bytes of user key) in
 226         * the resulting descriptor. Use DKP<ptr,imm> instead => both
 227         * virtual and dma key addresses are needed.
 228         */
 229        ctx->adata.key_virt = ctx->key;
 230        ctx->adata.key_dma = ctx->key_dma;
 231
 232        ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 233        ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 234
 235        data_len[0] = ctx->adata.keylen_pad;
 236        data_len[1] = ctx->cdata.keylen;
 237
 238        if (alg->caam.geniv)
 239                goto skip_enc;
 240
 241        /*
 242         * Job Descriptor and Shared Descriptors
 243         * must all fit into the 64-word Descriptor h/w Buffer
 244         */
 245        if (desc_inline_query(DESC_AEAD_ENC_LEN +
 246                              (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 247                              AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 248                              ARRAY_SIZE(data_len)) < 0)
 249                return -EINVAL;
 250
 251        ctx->adata.key_inline = !!(inl_mask & 1);
 252        ctx->cdata.key_inline = !!(inl_mask & 2);
 253
 254        /* aead_encrypt shared descriptor */
 255        desc = ctx->sh_desc_enc;
 256        cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
 257                               ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
 258                               false, ctrlpriv->era);
 259        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 260                                   desc_bytes(desc), ctx->dir);
 261
 262skip_enc:
 263        /*
 264         * Job Descriptor and Shared Descriptors
 265         * must all fit into the 64-word Descriptor h/w Buffer
 266         */
 267        if (desc_inline_query(DESC_AEAD_DEC_LEN +
 268                              (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 269                              AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 270                              ARRAY_SIZE(data_len)) < 0)
 271                return -EINVAL;
 272
 273        ctx->adata.key_inline = !!(inl_mask & 1);
 274        ctx->cdata.key_inline = !!(inl_mask & 2);
 275
 276        /* aead_decrypt shared descriptor */
 277        desc = ctx->sh_desc_dec;
 278        cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
 279                               ctx->authsize, alg->caam.geniv, is_rfc3686,
 280                               nonce, ctx1_iv_off, false, ctrlpriv->era);
 281        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 282                                   desc_bytes(desc), ctx->dir);
 283
 284        if (!alg->caam.geniv)
 285                goto skip_givenc;
 286
 287        /*
 288         * Job Descriptor and Shared Descriptors
 289         * must all fit into the 64-word Descriptor h/w Buffer
 290         */
 291        if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
 292                              (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 293                              AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 294                              ARRAY_SIZE(data_len)) < 0)
 295                return -EINVAL;
 296
 297        ctx->adata.key_inline = !!(inl_mask & 1);
 298        ctx->cdata.key_inline = !!(inl_mask & 2);
 299
 300        /* aead_givencrypt shared descriptor */
 301        desc = ctx->sh_desc_enc;
 302        cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
 303                                  ctx->authsize, is_rfc3686, nonce,
 304                                  ctx1_iv_off, false, ctrlpriv->era);
 305        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 306                                   desc_bytes(desc), ctx->dir);
 307
 308skip_givenc:
 309        return 0;
 310}
 311
 312static int aead_setauthsize(struct crypto_aead *authenc,
 313                                    unsigned int authsize)
 314{
 315        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 316
 317        ctx->authsize = authsize;
 318        aead_set_sh_desc(authenc);
 319
 320        return 0;
 321}
 322
 323static int gcm_set_sh_desc(struct crypto_aead *aead)
 324{
 325        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 326        struct device *jrdev = ctx->jrdev;
 327        unsigned int ivsize = crypto_aead_ivsize(aead);
 328        u32 *desc;
 329        int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 330                        ctx->cdata.keylen;
 331
 332        if (!ctx->cdata.keylen || !ctx->authsize)
 333                return 0;
 334
 335        /*
 336         * AES GCM encrypt shared descriptor
 337         * Job Descriptor and Shared Descriptor
 338         * must fit into the 64-word Descriptor h/w Buffer
 339         */
 340        if (rem_bytes >= DESC_GCM_ENC_LEN) {
 341                ctx->cdata.key_inline = true;
 342                ctx->cdata.key_virt = ctx->key;
 343        } else {
 344                ctx->cdata.key_inline = false;
 345                ctx->cdata.key_dma = ctx->key_dma;
 346        }
 347
 348        desc = ctx->sh_desc_enc;
 349        cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
 350        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 351                                   desc_bytes(desc), ctx->dir);
 352
 353        /*
 354         * Job Descriptor and Shared Descriptors
 355         * must all fit into the 64-word Descriptor h/w Buffer
 356         */
 357        if (rem_bytes >= DESC_GCM_DEC_LEN) {
 358                ctx->cdata.key_inline = true;
 359                ctx->cdata.key_virt = ctx->key;
 360        } else {
 361                ctx->cdata.key_inline = false;
 362                ctx->cdata.key_dma = ctx->key_dma;
 363        }
 364
 365        desc = ctx->sh_desc_dec;
 366        cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
 367        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 368                                   desc_bytes(desc), ctx->dir);
 369
 370        return 0;
 371}
 372
 373static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 374{
 375        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 376        int err;
 377
 378        err = crypto_gcm_check_authsize(authsize);
 379        if (err)
 380                return err;
 381
 382        ctx->authsize = authsize;
 383        gcm_set_sh_desc(authenc);
 384
 385        return 0;
 386}
 387
 388static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 389{
 390        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 391        struct device *jrdev = ctx->jrdev;
 392        unsigned int ivsize = crypto_aead_ivsize(aead);
 393        u32 *desc;
 394        int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 395                        ctx->cdata.keylen;
 396
 397        if (!ctx->cdata.keylen || !ctx->authsize)
 398                return 0;
 399
 400        /*
 401         * RFC4106 encrypt shared descriptor
 402         * Job Descriptor and Shared Descriptor
 403         * must fit into the 64-word Descriptor h/w Buffer
 404         */
 405        if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
 406                ctx->cdata.key_inline = true;
 407                ctx->cdata.key_virt = ctx->key;
 408        } else {
 409                ctx->cdata.key_inline = false;
 410                ctx->cdata.key_dma = ctx->key_dma;
 411        }
 412
 413        desc = ctx->sh_desc_enc;
 414        cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 415                                  false);
 416        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 417                                   desc_bytes(desc), ctx->dir);
 418
 419        /*
 420         * Job Descriptor and Shared Descriptors
 421         * must all fit into the 64-word Descriptor h/w Buffer
 422         */
 423        if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
 424                ctx->cdata.key_inline = true;
 425                ctx->cdata.key_virt = ctx->key;
 426        } else {
 427                ctx->cdata.key_inline = false;
 428                ctx->cdata.key_dma = ctx->key_dma;
 429        }
 430
 431        desc = ctx->sh_desc_dec;
 432        cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 433                                  false);
 434        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 435                                   desc_bytes(desc), ctx->dir);
 436
 437        return 0;
 438}
 439
 440static int rfc4106_setauthsize(struct crypto_aead *authenc,
 441                               unsigned int authsize)
 442{
 443        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 444        int err;
 445
 446        err = crypto_rfc4106_check_authsize(authsize);
 447        if (err)
 448                return err;
 449
 450        ctx->authsize = authsize;
 451        rfc4106_set_sh_desc(authenc);
 452
 453        return 0;
 454}
 455
 456static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 457{
 458        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 459        struct device *jrdev = ctx->jrdev;
 460        unsigned int ivsize = crypto_aead_ivsize(aead);
 461        u32 *desc;
 462        int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 463                        ctx->cdata.keylen;
 464
 465        if (!ctx->cdata.keylen || !ctx->authsize)
 466                return 0;
 467
 468        /*
 469         * RFC4543 encrypt shared descriptor
 470         * Job Descriptor and Shared Descriptor
 471         * must fit into the 64-word Descriptor h/w Buffer
 472         */
 473        if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
 474                ctx->cdata.key_inline = true;
 475                ctx->cdata.key_virt = ctx->key;
 476        } else {
 477                ctx->cdata.key_inline = false;
 478                ctx->cdata.key_dma = ctx->key_dma;
 479        }
 480
 481        desc = ctx->sh_desc_enc;
 482        cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 483                                  false);
 484        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 485                                   desc_bytes(desc), ctx->dir);
 486
 487        /*
 488         * Job Descriptor and Shared Descriptors
 489         * must all fit into the 64-word Descriptor h/w Buffer
 490         */
 491        if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
 492                ctx->cdata.key_inline = true;
 493                ctx->cdata.key_virt = ctx->key;
 494        } else {
 495                ctx->cdata.key_inline = false;
 496                ctx->cdata.key_dma = ctx->key_dma;
 497        }
 498
 499        desc = ctx->sh_desc_dec;
 500        cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 501                                  false);
 502        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 503                                   desc_bytes(desc), ctx->dir);
 504
 505        return 0;
 506}
 507
 508static int rfc4543_setauthsize(struct crypto_aead *authenc,
 509                               unsigned int authsize)
 510{
 511        struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 512
 513        if (authsize != 16)
 514                return -EINVAL;
 515
 516        ctx->authsize = authsize;
 517        rfc4543_set_sh_desc(authenc);
 518
 519        return 0;
 520}
 521
 522static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 523{
 524        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 525        struct device *jrdev = ctx->jrdev;
 526        unsigned int ivsize = crypto_aead_ivsize(aead);
 527        u32 *desc;
 528
 529        if (!ctx->cdata.keylen || !ctx->authsize)
 530                return 0;
 531
 532        desc = ctx->sh_desc_enc;
 533        cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 534                               ctx->authsize, true, false);
 535        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 536                                   desc_bytes(desc), ctx->dir);
 537
 538        desc = ctx->sh_desc_dec;
 539        cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 540                               ctx->authsize, false, false);
 541        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 542                                   desc_bytes(desc), ctx->dir);
 543
 544        return 0;
 545}
 546
 547static int chachapoly_setauthsize(struct crypto_aead *aead,
 548                                  unsigned int authsize)
 549{
 550        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 551
 552        if (authsize != POLY1305_DIGEST_SIZE)
 553                return -EINVAL;
 554
 555        ctx->authsize = authsize;
 556        return chachapoly_set_sh_desc(aead);
 557}
 558
 559static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 560                             unsigned int keylen)
 561{
 562        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 563        unsigned int ivsize = crypto_aead_ivsize(aead);
 564        unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 565
 566        if (keylen != CHACHA_KEY_SIZE + saltlen)
 567                return -EINVAL;
 568
 569        ctx->cdata.key_virt = key;
 570        ctx->cdata.keylen = keylen - saltlen;
 571
 572        return chachapoly_set_sh_desc(aead);
 573}
 574
 575static int aead_setkey(struct crypto_aead *aead,
 576                               const u8 *key, unsigned int keylen)
 577{
 578        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 579        struct device *jrdev = ctx->jrdev;
 580        struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 581        struct crypto_authenc_keys keys;
 582        int ret = 0;
 583
 584        if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 585                goto badkey;
 586
 587        dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
 588               keys.authkeylen + keys.enckeylen, keys.enckeylen,
 589               keys.authkeylen);
 590        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 591                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 592
 593        /*
 594         * If DKP is supported, use it in the shared descriptor to generate
 595         * the split key.
 596         */
 597        if (ctrlpriv->era >= 6) {
 598                ctx->adata.keylen = keys.authkeylen;
 599                ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 600                                                      OP_ALG_ALGSEL_MASK);
 601
 602                if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 603                        goto badkey;
 604
 605                memcpy(ctx->key, keys.authkey, keys.authkeylen);
 606                memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
 607                       keys.enckeylen);
 608                dma_sync_single_for_device(jrdev, ctx->key_dma,
 609                                           ctx->adata.keylen_pad +
 610                                           keys.enckeylen, ctx->dir);
 611                goto skip_split_key;
 612        }
 613
 614        ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
 615                            keys.authkeylen, CAAM_MAX_KEY_SIZE -
 616                            keys.enckeylen);
 617        if (ret) {
 618                goto badkey;
 619        }
 620
 621        /* postpend encryption key to auth split key */
 622        memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 623        dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
 624                                   keys.enckeylen, ctx->dir);
 625
 626        print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
 627                             DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 628                             ctx->adata.keylen_pad + keys.enckeylen, 1);
 629
 630skip_split_key:
 631        ctx->cdata.keylen = keys.enckeylen;
 632        memzero_explicit(&keys, sizeof(keys));
 633        return aead_set_sh_desc(aead);
 634badkey:
 635        memzero_explicit(&keys, sizeof(keys));
 636        return -EINVAL;
 637}
 638
 639static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 640                            unsigned int keylen)
 641{
 642        struct crypto_authenc_keys keys;
 643        int err;
 644
 645        err = crypto_authenc_extractkeys(&keys, key, keylen);
 646        if (unlikely(err))
 647                return err;
 648
 649        err = verify_aead_des3_key(aead, keys.enckey, keys.enckeylen) ?:
 650              aead_setkey(aead, key, keylen);
 651
 652        memzero_explicit(&keys, sizeof(keys));
 653        return err;
 654}
 655
 656static int gcm_setkey(struct crypto_aead *aead,
 657                      const u8 *key, unsigned int keylen)
 658{
 659        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 660        struct device *jrdev = ctx->jrdev;
 661        int err;
 662
 663        err = aes_check_keylen(keylen);
 664        if (err)
 665                return err;
 666
 667        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 668                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 669
 670        memcpy(ctx->key, key, keylen);
 671        dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
 672        ctx->cdata.keylen = keylen;
 673
 674        return gcm_set_sh_desc(aead);
 675}
 676
 677static int rfc4106_setkey(struct crypto_aead *aead,
 678                          const u8 *key, unsigned int keylen)
 679{
 680        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 681        struct device *jrdev = ctx->jrdev;
 682        int err;
 683
 684        err = aes_check_keylen(keylen - 4);
 685        if (err)
 686                return err;
 687
 688        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 689                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 690
 691        memcpy(ctx->key, key, keylen);
 692
 693        /*
 694         * The last four bytes of the key material are used as the salt value
 695         * in the nonce. Update the AES key length.
 696         */
 697        ctx->cdata.keylen = keylen - 4;
 698        dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 699                                   ctx->dir);
 700        return rfc4106_set_sh_desc(aead);
 701}
 702
 703static int rfc4543_setkey(struct crypto_aead *aead,
 704                          const u8 *key, unsigned int keylen)
 705{
 706        struct caam_ctx *ctx = crypto_aead_ctx(aead);
 707        struct device *jrdev = ctx->jrdev;
 708        int err;
 709
 710        err = aes_check_keylen(keylen - 4);
 711        if (err)
 712                return err;
 713
 714        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 715                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 716
 717        memcpy(ctx->key, key, keylen);
 718
 719        /*
 720         * The last four bytes of the key material are used as the salt value
 721         * in the nonce. Update the AES key length.
 722         */
 723        ctx->cdata.keylen = keylen - 4;
 724        dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 725                                   ctx->dir);
 726        return rfc4543_set_sh_desc(aead);
 727}
 728
 729static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 730                           unsigned int keylen, const u32 ctx1_iv_off)
 731{
 732        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 733        struct caam_skcipher_alg *alg =
 734                container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
 735                             skcipher);
 736        struct device *jrdev = ctx->jrdev;
 737        unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 738        u32 *desc;
 739        const bool is_rfc3686 = alg->caam.rfc3686;
 740
 741        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
 742                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 743
 744        ctx->cdata.keylen = keylen;
 745        ctx->cdata.key_virt = key;
 746        ctx->cdata.key_inline = true;
 747
 748        /* skcipher_encrypt shared descriptor */
 749        desc = ctx->sh_desc_enc;
 750        cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 751                                   ctx1_iv_off);
 752        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 753                                   desc_bytes(desc), ctx->dir);
 754
 755        /* skcipher_decrypt shared descriptor */
 756        desc = ctx->sh_desc_dec;
 757        cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 758                                   ctx1_iv_off);
 759        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 760                                   desc_bytes(desc), ctx->dir);
 761
 762        return 0;
 763}
 764
 765static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
 766                               const u8 *key, unsigned int keylen)
 767{
 768        int err;
 769
 770        err = aes_check_keylen(keylen);
 771        if (err)
 772                return err;
 773
 774        return skcipher_setkey(skcipher, key, keylen, 0);
 775}
 776
 777static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
 778                                   const u8 *key, unsigned int keylen)
 779{
 780        u32 ctx1_iv_off;
 781        int err;
 782
 783        /*
 784         * RFC3686 specific:
 785         *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 786         *      | *key = {KEY, NONCE}
 787         */
 788        ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 789        keylen -= CTR_RFC3686_NONCE_SIZE;
 790
 791        err = aes_check_keylen(keylen);
 792        if (err)
 793                return err;
 794
 795        return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 796}
 797
 798static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
 799                               const u8 *key, unsigned int keylen)
 800{
 801        u32 ctx1_iv_off;
 802        int err;
 803
 804        /*
 805         * AES-CTR needs to load IV in CONTEXT1 reg
 806         * at an offset of 128bits (16bytes)
 807         * CONTEXT1[255:128] = IV
 808         */
 809        ctx1_iv_off = 16;
 810
 811        err = aes_check_keylen(keylen);
 812        if (err)
 813                return err;
 814
 815        return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
 816}
 817
 818static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
 819                               const u8 *key, unsigned int keylen)
 820{
 821        return verify_skcipher_des_key(skcipher, key) ?:
 822               skcipher_setkey(skcipher, key, keylen, 0);
 823}
 824
 825static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
 826                                const u8 *key, unsigned int keylen)
 827{
 828        return verify_skcipher_des3_key(skcipher, key) ?:
 829               skcipher_setkey(skcipher, key, keylen, 0);
 830}
 831
 832static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 833                               unsigned int keylen)
 834{
 835        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 836        struct device *jrdev = ctx->jrdev;
 837        struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 838        u32 *desc;
 839        int err;
 840
 841        err = xts_verify_key(skcipher, key, keylen);
 842        if (err) {
 843                dev_dbg(jrdev, "key size mismatch\n");
 844                return err;
 845        }
 846
 847        if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
 848                ctx->xts_key_fallback = true;
 849
 850        if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
 851                err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
 852                if (err)
 853                        return err;
 854        }
 855
 856        ctx->cdata.keylen = keylen;
 857        ctx->cdata.key_virt = key;
 858        ctx->cdata.key_inline = true;
 859
 860        /* xts_skcipher_encrypt shared descriptor */
 861        desc = ctx->sh_desc_enc;
 862        cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
 863        dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 864                                   desc_bytes(desc), ctx->dir);
 865
 866        /* xts_skcipher_decrypt shared descriptor */
 867        desc = ctx->sh_desc_dec;
 868        cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
 869        dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 870                                   desc_bytes(desc), ctx->dir);
 871
 872        return 0;
 873}
 874
 875/*
 876 * aead_edesc - s/w-extended aead descriptor
 877 * @src_nents: number of segments in input s/w scatterlist
 878 * @dst_nents: number of segments in output s/w scatterlist
 879 * @mapped_src_nents: number of segments in input h/w link table
 880 * @mapped_dst_nents: number of segments in output h/w link table
 881 * @sec4_sg_bytes: length of dma mapped sec4_sg space
 882 * @bklog: stored to determine if the request needs backlog
 883 * @sec4_sg_dma: bus physical mapped address of h/w link table
 884 * @sec4_sg: pointer to h/w link table
 885 * @hw_desc: the h/w job descriptor followed by any referenced link tables
 886 */
 887struct aead_edesc {
 888        int src_nents;
 889        int dst_nents;
 890        int mapped_src_nents;
 891        int mapped_dst_nents;
 892        int sec4_sg_bytes;
 893        bool bklog;
 894        dma_addr_t sec4_sg_dma;
 895        struct sec4_sg_entry *sec4_sg;
 896        u32 hw_desc[];
 897};
 898
 899/*
 900 * skcipher_edesc - s/w-extended skcipher descriptor
 901 * @src_nents: number of segments in input s/w scatterlist
 902 * @dst_nents: number of segments in output s/w scatterlist
 903 * @mapped_src_nents: number of segments in input h/w link table
 904 * @mapped_dst_nents: number of segments in output h/w link table
 905 * @iv_dma: dma address of iv for checking continuity and link table
 906 * @sec4_sg_bytes: length of dma mapped sec4_sg space
 907 * @bklog: stored to determine if the request needs backlog
 908 * @sec4_sg_dma: bus physical mapped address of h/w link table
 909 * @sec4_sg: pointer to h/w link table
 910 * @hw_desc: the h/w job descriptor followed by any referenced link tables
 911 *           and IV
 912 */
 913struct skcipher_edesc {
 914        int src_nents;
 915        int dst_nents;
 916        int mapped_src_nents;
 917        int mapped_dst_nents;
 918        dma_addr_t iv_dma;
 919        int sec4_sg_bytes;
 920        bool bklog;
 921        dma_addr_t sec4_sg_dma;
 922        struct sec4_sg_entry *sec4_sg;
 923        u32 hw_desc[];
 924};
 925
 926static void caam_unmap(struct device *dev, struct scatterlist *src,
 927                       struct scatterlist *dst, int src_nents,
 928                       int dst_nents,
 929                       dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
 930                       int sec4_sg_bytes)
 931{
 932        if (dst != src) {
 933                if (src_nents)
 934                        dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 935                if (dst_nents)
 936                        dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 937        } else {
 938                dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 939        }
 940
 941        if (iv_dma)
 942                dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
 943        if (sec4_sg_bytes)
 944                dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
 945                                 DMA_TO_DEVICE);
 946}
 947
 948static void aead_unmap(struct device *dev,
 949                       struct aead_edesc *edesc,
 950                       struct aead_request *req)
 951{
 952        caam_unmap(dev, req->src, req->dst,
 953                   edesc->src_nents, edesc->dst_nents, 0, 0,
 954                   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 955}
 956
 957static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
 958                           struct skcipher_request *req)
 959{
 960        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
 961        int ivsize = crypto_skcipher_ivsize(skcipher);
 962
 963        caam_unmap(dev, req->src, req->dst,
 964                   edesc->src_nents, edesc->dst_nents,
 965                   edesc->iv_dma, ivsize,
 966                   edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 967}
 968
 969static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
 970                            void *context)
 971{
 972        struct aead_request *req = context;
 973        struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
 974        struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
 975        struct aead_edesc *edesc;
 976        int ecode = 0;
 977        bool has_bklog;
 978
 979        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 980
 981        edesc = rctx->edesc;
 982        has_bklog = edesc->bklog;
 983
 984        if (err)
 985                ecode = caam_jr_strstatus(jrdev, err);
 986
 987        aead_unmap(jrdev, edesc, req);
 988
 989        kfree(edesc);
 990
 991        /*
 992         * If no backlog flag, the completion of the request is done
 993         * by CAAM, not crypto engine.
 994         */
 995        if (!has_bklog)
 996                aead_request_complete(req, ecode);
 997        else
 998                crypto_finalize_aead_request(jrp->engine, req, ecode);
 999}
1000
1001static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
1002                                void *context)
1003{
1004        struct skcipher_request *req = context;
1005        struct skcipher_edesc *edesc;
1006        struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1007        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1008        struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
1009        int ivsize = crypto_skcipher_ivsize(skcipher);
1010        int ecode = 0;
1011        bool has_bklog;
1012
1013        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1014
1015        edesc = rctx->edesc;
1016        has_bklog = edesc->bklog;
1017        if (err)
1018                ecode = caam_jr_strstatus(jrdev, err);
1019
1020        skcipher_unmap(jrdev, edesc, req);
1021
1022        /*
1023         * The crypto API expects us to set the IV (req->iv) to the last
1024         * ciphertext block (CBC mode) or last counter (CTR mode).
1025         * This is used e.g. by the CTS mode.
1026         */
1027        if (ivsize && !ecode) {
1028                memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
1029                       ivsize);
1030
1031                print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
1032                                     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1033                                     ivsize, 1);
1034        }
1035
1036        caam_dump_sg("dst    @" __stringify(__LINE__)": ",
1037                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1038                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1039
1040        kfree(edesc);
1041
1042        /*
1043         * If no backlog flag, the completion of the request is done
1044         * by CAAM, not crypto engine.
1045         */
1046        if (!has_bklog)
1047                skcipher_request_complete(req, ecode);
1048        else
1049                crypto_finalize_skcipher_request(jrp->engine, req, ecode);
1050}
1051
1052/*
1053 * Fill in aead job descriptor
1054 */
1055static void init_aead_job(struct aead_request *req,
1056                          struct aead_edesc *edesc,
1057                          bool all_contig, bool encrypt)
1058{
1059        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1060        struct caam_ctx *ctx = crypto_aead_ctx(aead);
1061        int authsize = ctx->authsize;
1062        u32 *desc = edesc->hw_desc;
1063        u32 out_options, in_options;
1064        dma_addr_t dst_dma, src_dma;
1065        int len, sec4_sg_index = 0;
1066        dma_addr_t ptr;
1067        u32 *sh_desc;
1068
1069        sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1070        ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1071
1072        len = desc_len(sh_desc);
1073        init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1074
1075        if (all_contig) {
1076                src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :
1077                                                    0;
1078                in_options = 0;
1079        } else {
1080                src_dma = edesc->sec4_sg_dma;
1081                sec4_sg_index += edesc->mapped_src_nents;
1082                in_options = LDST_SGF;
1083        }
1084
1085        append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
1086                          in_options);
1087
1088        dst_dma = src_dma;
1089        out_options = in_options;
1090
1091        if (unlikely(req->src != req->dst)) {
1092                if (!edesc->mapped_dst_nents) {
1093                        dst_dma = 0;
1094                        out_options = 0;
1095                } else if (edesc->mapped_dst_nents == 1) {
1096                        dst_dma = sg_dma_address(req->dst);
1097                        out_options = 0;
1098                } else {
1099                        dst_dma = edesc->sec4_sg_dma +
1100                                  sec4_sg_index *
1101                                  sizeof(struct sec4_sg_entry);
1102                        out_options = LDST_SGF;
1103                }
1104        }
1105
1106        if (encrypt)
1107                append_seq_out_ptr(desc, dst_dma,
1108                                   req->assoclen + req->cryptlen + authsize,
1109                                   out_options);
1110        else
1111                append_seq_out_ptr(desc, dst_dma,
1112                                   req->assoclen + req->cryptlen - authsize,
1113                                   out_options);
1114}
1115
1116static void init_gcm_job(struct aead_request *req,
1117                         struct aead_edesc *edesc,
1118                         bool all_contig, bool encrypt)
1119{
1120        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1121        struct caam_ctx *ctx = crypto_aead_ctx(aead);
1122        unsigned int ivsize = crypto_aead_ivsize(aead);
1123        u32 *desc = edesc->hw_desc;
1124        bool generic_gcm = (ivsize == GCM_AES_IV_SIZE);
1125        unsigned int last;
1126
1127        init_aead_job(req, edesc, all_contig, encrypt);
1128        append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1129
1130        /* BUG This should not be specific to generic GCM. */
1131        last = 0;
1132        if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1133                last = FIFOLD_TYPE_LAST1;
1134
1135        /* Read GCM IV */
1136        append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1137                         FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last);
1138        /* Append Salt */
1139        if (!generic_gcm)
1140                append_data(desc, ctx->key + ctx->cdata.keylen, 4);
1141        /* Append IV */
1142        append_data(desc, req->iv, ivsize);
1143        /* End of blank commands */
1144}
1145
1146static void init_chachapoly_job(struct aead_request *req,
1147                                struct aead_edesc *edesc, bool all_contig,
1148                                bool encrypt)
1149{
1150        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1151        unsigned int ivsize = crypto_aead_ivsize(aead);
1152        unsigned int assoclen = req->assoclen;
1153        u32 *desc = edesc->hw_desc;
1154        u32 ctx_iv_off = 4;
1155
1156        init_aead_job(req, edesc, all_contig, encrypt);
1157
1158        if (ivsize != CHACHAPOLY_IV_SIZE) {
1159                /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
1160                ctx_iv_off += 4;
1161
1162                /*
1163                 * The associated data comes already with the IV but we need
1164                 * to skip it when we authenticate or encrypt...
1165                 */
1166                assoclen -= ivsize;
1167        }
1168
1169        append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen);
1170
1171        /*
1172         * For IPsec load the IV further in the same register.
1173         * For RFC7539 simply load the 12 bytes nonce in a single operation
1174         */
1175        append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB |
1176                           LDST_SRCDST_BYTE_CONTEXT |
1177                           ctx_iv_off << LDST_OFFSET_SHIFT);
1178}
1179
1180static void init_authenc_job(struct aead_request *req,
1181                             struct aead_edesc *edesc,
1182                             bool all_contig, bool encrypt)
1183{
1184        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1185        struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
1186                                                 struct caam_aead_alg, aead);
1187        unsigned int ivsize = crypto_aead_ivsize(aead);
1188        struct caam_ctx *ctx = crypto_aead_ctx(aead);
1189        struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
1190        const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
1191                               OP_ALG_AAI_CTR_MOD128);
1192        const bool is_rfc3686 = alg->caam.rfc3686;
1193        u32 *desc = edesc->hw_desc;
1194        u32 ivoffset = 0;
1195
1196        /*
1197         * AES-CTR needs to load IV in CONTEXT1 reg
1198         * at an offset of 128bits (16bytes)
1199         * CONTEXT1[255:128] = IV
1200         */
1201        if (ctr_mode)
1202                ivoffset = 16;
1203
1204        /*
1205         * RFC3686 specific:
1206         *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1207         */
1208        if (is_rfc3686)
1209                ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
1210
1211        init_aead_job(req, edesc, all_contig, encrypt);
1212
1213        /*
1214         * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1215         * having DPOVRD as destination.
1216         */
1217        if (ctrlpriv->era < 3)
1218                append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1219        else
1220                append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
1221
1222        if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
1223                append_load_as_imm(desc, req->iv, ivsize,
1224                                   LDST_CLASS_1_CCB |
1225                                   LDST_SRCDST_BYTE_CONTEXT |
1226                                   (ivoffset << LDST_OFFSET_SHIFT));
1227}
1228
1229/*
1230 * Fill in skcipher job descriptor
1231 */
1232static void init_skcipher_job(struct skcipher_request *req,
1233                              struct skcipher_edesc *edesc,
1234                              const bool encrypt)
1235{
1236        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1237        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1238        struct device *jrdev = ctx->jrdev;
1239        int ivsize = crypto_skcipher_ivsize(skcipher);
1240        u32 *desc = edesc->hw_desc;
1241        u32 *sh_desc;
1242        u32 in_options = 0, out_options = 0;
1243        dma_addr_t src_dma, dst_dma, ptr;
1244        int len, sec4_sg_index = 0;
1245
1246        print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
1247                             DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1248        dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
1249               (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
1250
1251        caam_dump_sg("src    @" __stringify(__LINE__)": ",
1252                     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1253                     edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1254
1255        sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1256        ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1257
1258        len = desc_len(sh_desc);
1259        init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1260
1261        if (ivsize || edesc->mapped_src_nents > 1) {
1262                src_dma = edesc->sec4_sg_dma;
1263                sec4_sg_index = edesc->mapped_src_nents + !!ivsize;
1264                in_options = LDST_SGF;
1265        } else {
1266                src_dma = sg_dma_address(req->src);
1267        }
1268
1269        append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options);
1270
1271        if (likely(req->src == req->dst)) {
1272                dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
1273                out_options = in_options;
1274        } else if (!ivsize && edesc->mapped_dst_nents == 1) {
1275                dst_dma = sg_dma_address(req->dst);
1276        } else {
1277                dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
1278                          sizeof(struct sec4_sg_entry);
1279                out_options = LDST_SGF;
1280        }
1281
1282        append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options);
1283}
1284
1285/*
1286 * allocate and map the aead extended descriptor
1287 */
1288static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1289                                           int desc_bytes, bool *all_contig_ptr,
1290                                           bool encrypt)
1291{
1292        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1293        struct caam_ctx *ctx = crypto_aead_ctx(aead);
1294        struct device *jrdev = ctx->jrdev;
1295        struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1296        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1297                       GFP_KERNEL : GFP_ATOMIC;
1298        int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1299        int src_len, dst_len = 0;
1300        struct aead_edesc *edesc;
1301        int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
1302        unsigned int authsize = ctx->authsize;
1303
1304        if (unlikely(req->dst != req->src)) {
1305                src_len = req->assoclen + req->cryptlen;
1306                dst_len = src_len + (encrypt ? authsize : (-authsize));
1307
1308                src_nents = sg_nents_for_len(req->src, src_len);
1309                if (unlikely(src_nents < 0)) {
1310                        dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1311                                src_len);
1312                        return ERR_PTR(src_nents);
1313                }
1314
1315                dst_nents = sg_nents_for_len(req->dst, dst_len);
1316                if (unlikely(dst_nents < 0)) {
1317                        dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1318                                dst_len);
1319                        return ERR_PTR(dst_nents);
1320                }
1321        } else {
1322                src_len = req->assoclen + req->cryptlen +
1323                          (encrypt ? authsize : 0);
1324
1325                src_nents = sg_nents_for_len(req->src, src_len);
1326                if (unlikely(src_nents < 0)) {
1327                        dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1328                                src_len);
1329                        return ERR_PTR(src_nents);
1330                }
1331        }
1332
1333        if (likely(req->src == req->dst)) {
1334                mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1335                                              DMA_BIDIRECTIONAL);
1336                if (unlikely(!mapped_src_nents)) {
1337                        dev_err(jrdev, "unable to map source\n");
1338                        return ERR_PTR(-ENOMEM);
1339                }
1340        } else {
1341                /* Cover also the case of null (zero length) input data */
1342                if (src_nents) {
1343                        mapped_src_nents = dma_map_sg(jrdev, req->src,
1344                                                      src_nents, DMA_TO_DEVICE);
1345                        if (unlikely(!mapped_src_nents)) {
1346                                dev_err(jrdev, "unable to map source\n");
1347                                return ERR_PTR(-ENOMEM);
1348                        }
1349                } else {
1350                        mapped_src_nents = 0;
1351                }
1352
1353                /* Cover also the case of null (zero length) output data */
1354                if (dst_nents) {
1355                        mapped_dst_nents = dma_map_sg(jrdev, req->dst,
1356                                                      dst_nents,
1357                                                      DMA_FROM_DEVICE);
1358                        if (unlikely(!mapped_dst_nents)) {
1359                                dev_err(jrdev, "unable to map destination\n");
1360                                dma_unmap_sg(jrdev, req->src, src_nents,
1361                                             DMA_TO_DEVICE);
1362                                return ERR_PTR(-ENOMEM);
1363                        }
1364                } else {
1365                        mapped_dst_nents = 0;
1366                }
1367        }
1368
1369        /*
1370         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1371         * the end of the table by allocating more S/G entries.
1372         */
1373        sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
1374        if (mapped_dst_nents > 1)
1375                sec4_sg_len += pad_sg_nents(mapped_dst_nents);
1376        else
1377                sec4_sg_len = pad_sg_nents(sec4_sg_len);
1378
1379        sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1380
1381        /* allocate space for base edesc and hw desc commands, link tables */
1382        edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1383                        GFP_DMA | flags);
1384        if (!edesc) {
1385                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1386                           0, 0, 0);
1387                return ERR_PTR(-ENOMEM);
1388        }
1389
1390        edesc->src_nents = src_nents;
1391        edesc->dst_nents = dst_nents;
1392        edesc->mapped_src_nents = mapped_src_nents;
1393        edesc->mapped_dst_nents = mapped_dst_nents;
1394        edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1395                         desc_bytes;
1396
1397        rctx->edesc = edesc;
1398
1399        *all_contig_ptr = !(mapped_src_nents > 1);
1400
1401        sec4_sg_index = 0;
1402        if (mapped_src_nents > 1) {
1403                sg_to_sec4_sg_last(req->src, src_len,
1404                                   edesc->sec4_sg + sec4_sg_index, 0);
1405                sec4_sg_index += mapped_src_nents;
1406        }
1407        if (mapped_dst_nents > 1) {
1408                sg_to_sec4_sg_last(req->dst, dst_len,
1409                                   edesc->sec4_sg + sec4_sg_index, 0);
1410        }
1411
1412        if (!sec4_sg_bytes)
1413                return edesc;
1414
1415        edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1416                                            sec4_sg_bytes, DMA_TO_DEVICE);
1417        if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1418                dev_err(jrdev, "unable to map S/G table\n");
1419                aead_unmap(jrdev, edesc, req);
1420                kfree(edesc);
1421                return ERR_PTR(-ENOMEM);
1422        }
1423
1424        edesc->sec4_sg_bytes = sec4_sg_bytes;
1425
1426        return edesc;
1427}
1428
1429static int aead_enqueue_req(struct device *jrdev, struct aead_request *req)
1430{
1431        struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
1432        struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1433        struct aead_edesc *edesc = rctx->edesc;
1434        u32 *desc = edesc->hw_desc;
1435        int ret;
1436
1437        /*
1438         * Only the backlog request are sent to crypto-engine since the others
1439         * can be handled by CAAM, if free, especially since JR has up to 1024
1440         * entries (more than the 10 entries from crypto-engine).
1441         */
1442        if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1443                ret = crypto_transfer_aead_request_to_engine(jrpriv->engine,
1444                                                             req);
1445        else
1446                ret = caam_jr_enqueue(jrdev, desc, aead_crypt_done, req);
1447
1448        if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
1449                aead_unmap(jrdev, edesc, req);
1450                kfree(rctx->edesc);
1451        }
1452
1453        return ret;
1454}
1455
1456static inline int chachapoly_crypt(struct aead_request *req, bool encrypt)
1457{
1458        struct aead_edesc *edesc;
1459        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1460        struct caam_ctx *ctx = crypto_aead_ctx(aead);
1461        struct device *jrdev = ctx->jrdev;
1462        bool all_contig;
1463        u32 *desc;
1464
1465        edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
1466                                 encrypt);
1467        if (IS_ERR(edesc))
1468                return PTR_ERR(edesc);
1469
1470        desc = edesc->hw_desc;
1471
1472        init_chachapoly_job(req, edesc, all_contig, encrypt);
1473        print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
1474                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1475                             1);
1476
1477        return aead_enqueue_req(jrdev, req);
1478}
1479
1480static int chachapoly_encrypt(struct aead_request *req)
1481{
1482        return chachapoly_crypt(req, true);
1483}
1484
1485static int chachapoly_decrypt(struct aead_request *req)
1486{
1487        return chachapoly_crypt(req, false);
1488}
1489
1490static inline int aead_crypt(struct aead_request *req, bool encrypt)
1491{
1492        struct aead_edesc *edesc;
1493        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1494        struct caam_ctx *ctx = crypto_aead_ctx(aead);
1495        struct device *jrdev = ctx->jrdev;
1496        bool all_contig;
1497
1498        /* allocate extended descriptor */
1499        edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1500                                 &all_contig, encrypt);
1501        if (IS_ERR(edesc))
1502                return PTR_ERR(edesc);
1503
1504        /* Create and submit job descriptor */
1505        init_authenc_job(req, edesc, all_contig, encrypt);
1506
1507        print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1508                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1509                             desc_bytes(edesc->hw_desc), 1);
1510
1511        return aead_enqueue_req(jrdev, req);
1512}
1513
1514static int aead_encrypt(struct aead_request *req)
1515{
1516        return aead_crypt(req, true);
1517}
1518
1519static int aead_decrypt(struct aead_request *req)
1520{
1521        return aead_crypt(req, false);
1522}
1523
1524static int aead_do_one_req(struct crypto_engine *engine, void *areq)
1525{
1526        struct aead_request *req = aead_request_cast(areq);
1527        struct caam_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
1528        struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1529        u32 *desc = rctx->edesc->hw_desc;
1530        int ret;
1531
1532        rctx->edesc->bklog = true;
1533
1534        ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
1535
1536        if (ret != -EINPROGRESS) {
1537                aead_unmap(ctx->jrdev, rctx->edesc, req);
1538                kfree(rctx->edesc);
1539        } else {
1540                ret = 0;
1541        }
1542
1543        return ret;
1544}
1545
1546static inline int gcm_crypt(struct aead_request *req, bool encrypt)
1547{
1548        struct aead_edesc *edesc;
1549        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1550        struct caam_ctx *ctx = crypto_aead_ctx(aead);
1551        struct device *jrdev = ctx->jrdev;
1552        bool all_contig;
1553
1554        /* allocate extended descriptor */
1555        edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig,
1556                                 encrypt);
1557        if (IS_ERR(edesc))
1558                return PTR_ERR(edesc);
1559
1560        /* Create and submit job descriptor */
1561        init_gcm_job(req, edesc, all_contig, encrypt);
1562
1563        print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1564                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1565                             desc_bytes(edesc->hw_desc), 1);
1566
1567        return aead_enqueue_req(jrdev, req);
1568}
1569
1570static int gcm_encrypt(struct aead_request *req)
1571{
1572        return gcm_crypt(req, true);
1573}
1574
1575static int gcm_decrypt(struct aead_request *req)
1576{
1577        return gcm_crypt(req, false);
1578}
1579
1580static int ipsec_gcm_encrypt(struct aead_request *req)
1581{
1582        return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_encrypt(req);
1583}
1584
1585static int ipsec_gcm_decrypt(struct aead_request *req)
1586{
1587        return crypto_ipsec_check_assoclen(req->assoclen) ? : gcm_decrypt(req);
1588}
1589
1590/*
1591 * allocate and map the skcipher extended descriptor for skcipher
1592 */
1593static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1594                                                   int desc_bytes)
1595{
1596        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1597        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1598        struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1599        struct device *jrdev = ctx->jrdev;
1600        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1601                       GFP_KERNEL : GFP_ATOMIC;
1602        int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1603        struct skcipher_edesc *edesc;
1604        dma_addr_t iv_dma = 0;
1605        u8 *iv;
1606        int ivsize = crypto_skcipher_ivsize(skcipher);
1607        int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1608
1609        src_nents = sg_nents_for_len(req->src, req->cryptlen);
1610        if (unlikely(src_nents < 0)) {
1611                dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1612                        req->cryptlen);
1613                return ERR_PTR(src_nents);
1614        }
1615
1616        if (req->dst != req->src) {
1617                dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1618                if (unlikely(dst_nents < 0)) {
1619                        dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1620                                req->cryptlen);
1621                        return ERR_PTR(dst_nents);
1622                }
1623        }
1624
1625        if (likely(req->src == req->dst)) {
1626                mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1627                                              DMA_BIDIRECTIONAL);
1628                if (unlikely(!mapped_src_nents)) {
1629                        dev_err(jrdev, "unable to map source\n");
1630                        return ERR_PTR(-ENOMEM);
1631                }
1632        } else {
1633                mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1634                                              DMA_TO_DEVICE);
1635                if (unlikely(!mapped_src_nents)) {
1636                        dev_err(jrdev, "unable to map source\n");
1637                        return ERR_PTR(-ENOMEM);
1638                }
1639                mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1640                                              DMA_FROM_DEVICE);
1641                if (unlikely(!mapped_dst_nents)) {
1642                        dev_err(jrdev, "unable to map destination\n");
1643                        dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1644                        return ERR_PTR(-ENOMEM);
1645                }
1646        }
1647
1648        if (!ivsize && mapped_src_nents == 1)
1649                sec4_sg_ents = 0; // no need for an input hw s/g table
1650        else
1651                sec4_sg_ents = mapped_src_nents + !!ivsize;
1652        dst_sg_idx = sec4_sg_ents;
1653
1654        /*
1655         * Input, output HW S/G tables: [IV, src][dst, IV]
1656         * IV entries point to the same buffer
1657         * If src == dst, S/G entries are reused (S/G tables overlap)
1658         *
1659         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1660         * the end of the table by allocating more S/G entries. Logic:
1661         * if (output S/G)
1662         *      pad output S/G, if needed
1663         * else if (input S/G) ...
1664         *      pad input S/G, if needed
1665         */
1666        if (ivsize || mapped_dst_nents > 1) {
1667                if (req->src == req->dst)
1668                        sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents);
1669                else
1670                        sec4_sg_ents += pad_sg_nents(mapped_dst_nents +
1671                                                     !!ivsize);
1672        } else {
1673                sec4_sg_ents = pad_sg_nents(sec4_sg_ents);
1674        }
1675
1676        sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1677
1678        /*
1679         * allocate space for base edesc and hw desc commands, link tables, IV
1680         */
1681        edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
1682                        GFP_DMA | flags);
1683        if (!edesc) {
1684                dev_err(jrdev, "could not allocate extended descriptor\n");
1685                caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1686                           0, 0, 0);
1687                return ERR_PTR(-ENOMEM);
1688        }
1689
1690        edesc->src_nents = src_nents;
1691        edesc->dst_nents = dst_nents;
1692        edesc->mapped_src_nents = mapped_src_nents;
1693        edesc->mapped_dst_nents = mapped_dst_nents;
1694        edesc->sec4_sg_bytes = sec4_sg_bytes;
1695        edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
1696                                                  desc_bytes);
1697        rctx->edesc = edesc;
1698
1699        /* Make sure IV is located in a DMAable area */
1700        if (ivsize) {
1701                iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
1702                memcpy(iv, req->iv, ivsize);
1703
1704                iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
1705                if (dma_mapping_error(jrdev, iv_dma)) {
1706                        dev_err(jrdev, "unable to map IV\n");
1707                        caam_unmap(jrdev, req->src, req->dst, src_nents,
1708                                   dst_nents, 0, 0, 0, 0);
1709                        kfree(edesc);
1710                        return ERR_PTR(-ENOMEM);
1711                }
1712
1713                dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1714        }
1715        if (dst_sg_idx)
1716                sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +
1717                              !!ivsize, 0);
1718
1719        if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
1720                sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +
1721                              dst_sg_idx, 0);
1722
1723        if (ivsize)
1724                dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +
1725                                   mapped_dst_nents, iv_dma, ivsize, 0);
1726
1727        if (ivsize || mapped_dst_nents > 1)
1728                sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +
1729                                    mapped_dst_nents - 1 + !!ivsize);
1730
1731        if (sec4_sg_bytes) {
1732                edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1733                                                    sec4_sg_bytes,
1734                                                    DMA_TO_DEVICE);
1735                if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1736                        dev_err(jrdev, "unable to map S/G table\n");
1737                        caam_unmap(jrdev, req->src, req->dst, src_nents,
1738                                   dst_nents, iv_dma, ivsize, 0, 0);
1739                        kfree(edesc);
1740                        return ERR_PTR(-ENOMEM);
1741                }
1742        }
1743
1744        edesc->iv_dma = iv_dma;
1745
1746        print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
1747                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1748                             sec4_sg_bytes, 1);
1749
1750        return edesc;
1751}
1752
1753static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
1754{
1755        struct skcipher_request *req = skcipher_request_cast(areq);
1756        struct caam_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1757        struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1758        u32 *desc = rctx->edesc->hw_desc;
1759        int ret;
1760
1761        rctx->edesc->bklog = true;
1762
1763        ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
1764
1765        if (ret != -EINPROGRESS) {
1766                skcipher_unmap(ctx->jrdev, rctx->edesc, req);
1767                kfree(rctx->edesc);
1768        } else {
1769                ret = 0;
1770        }
1771
1772        return ret;
1773}
1774
1775static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1776{
1777        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1778        unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
1779
1780        return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1781}
1782
1783static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1784{
1785        struct skcipher_edesc *edesc;
1786        struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1787        struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1788        struct device *jrdev = ctx->jrdev;
1789        struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
1790        struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
1791        u32 *desc;
1792        int ret = 0;
1793
1794        /*
1795         * XTS is expected to return an error even for input length = 0
1796         * Note that the case input length < block size will be caught during
1797         * HW offloading and return an error.
1798         */
1799        if (!req->cryptlen && !ctx->fallback)
1800                return 0;
1801
1802        if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
1803                              ctx->xts_key_fallback)) {
1804                struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1805
1806                skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
1807                skcipher_request_set_callback(&rctx->fallback_req,
1808                                              req->base.flags,
1809                                              req->base.complete,
1810                                              req->base.data);
1811                skcipher_request_set_crypt(&rctx->fallback_req, req->src,
1812                                           req->dst, req->cryptlen, req->iv);
1813
1814                return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
1815                                 crypto_skcipher_decrypt(&rctx->fallback_req);
1816        }
1817
1818        /* allocate extended descriptor */
1819        edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
1820        if (IS_ERR(edesc))
1821                return PTR_ERR(edesc);
1822
1823        /* Create and submit job descriptor*/
1824        init_skcipher_job(req, edesc, encrypt);
1825
1826        print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
1827                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1828                             desc_bytes(edesc->hw_desc), 1);
1829
1830        desc = edesc->hw_desc;
1831        /*
1832         * Only the backlog request are sent to crypto-engine since the others
1833         * can be handled by CAAM, if free, especially since JR has up to 1024
1834         * entries (more than the 10 entries from crypto-engine).
1835         */
1836        if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1837                ret = crypto_transfer_skcipher_request_to_engine(jrpriv->engine,
1838                                                                 req);
1839        else
1840                ret = caam_jr_enqueue(jrdev, desc, skcipher_crypt_done, req);
1841
1842        if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
1843                skcipher_unmap(jrdev, edesc, req);
1844                kfree(edesc);
1845        }
1846
1847        return ret;
1848}
1849
1850static int skcipher_encrypt(struct skcipher_request *req)
1851{
1852        return skcipher_crypt(req, true);
1853}
1854
1855static int skcipher_decrypt(struct skcipher_request *req)
1856{
1857        return skcipher_crypt(req, false);
1858}
1859
1860static struct caam_skcipher_alg driver_algs[] = {
1861        {
1862                .skcipher = {
1863                        .base = {
1864                                .cra_name = "cbc(aes)",
1865                                .cra_driver_name = "cbc-aes-caam",
1866                                .cra_blocksize = AES_BLOCK_SIZE,
1867                        },
1868                        .setkey = aes_skcipher_setkey,
1869                        .encrypt = skcipher_encrypt,
1870                        .decrypt = skcipher_decrypt,
1871                        .min_keysize = AES_MIN_KEY_SIZE,
1872                        .max_keysize = AES_MAX_KEY_SIZE,
1873                        .ivsize = AES_BLOCK_SIZE,
1874                },
1875                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1876        },
1877        {
1878                .skcipher = {
1879                        .base = {
1880                                .cra_name = "cbc(des3_ede)",
1881                                .cra_driver_name = "cbc-3des-caam",
1882                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1883                        },
1884                        .setkey = des3_skcipher_setkey,
1885                        .encrypt = skcipher_encrypt,
1886                        .decrypt = skcipher_decrypt,
1887                        .min_keysize = DES3_EDE_KEY_SIZE,
1888                        .max_keysize = DES3_EDE_KEY_SIZE,
1889                        .ivsize = DES3_EDE_BLOCK_SIZE,
1890                },
1891                .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1892        },
1893        {
1894                .skcipher = {
1895                        .base = {
1896                                .cra_name = "cbc(des)",
1897                                .cra_driver_name = "cbc-des-caam",
1898                                .cra_blocksize = DES_BLOCK_SIZE,
1899                        },
1900                        .setkey = des_skcipher_setkey,
1901                        .encrypt = skcipher_encrypt,
1902                        .decrypt = skcipher_decrypt,
1903                        .min_keysize = DES_KEY_SIZE,
1904                        .max_keysize = DES_KEY_SIZE,
1905                        .ivsize = DES_BLOCK_SIZE,
1906                },
1907                .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1908        },
1909        {
1910                .skcipher = {
1911                        .base = {
1912                                .cra_name = "ctr(aes)",
1913                                .cra_driver_name = "ctr-aes-caam",
1914                                .cra_blocksize = 1,
1915                        },
1916                        .setkey = ctr_skcipher_setkey,
1917                        .encrypt = skcipher_encrypt,
1918                        .decrypt = skcipher_decrypt,
1919                        .min_keysize = AES_MIN_KEY_SIZE,
1920                        .max_keysize = AES_MAX_KEY_SIZE,
1921                        .ivsize = AES_BLOCK_SIZE,
1922                        .chunksize = AES_BLOCK_SIZE,
1923                },
1924                .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1925                                        OP_ALG_AAI_CTR_MOD128,
1926        },
1927        {
1928                .skcipher = {
1929                        .base = {
1930                                .cra_name = "rfc3686(ctr(aes))",
1931                                .cra_driver_name = "rfc3686-ctr-aes-caam",
1932                                .cra_blocksize = 1,
1933                        },
1934                        .setkey = rfc3686_skcipher_setkey,
1935                        .encrypt = skcipher_encrypt,
1936                        .decrypt = skcipher_decrypt,
1937                        .min_keysize = AES_MIN_KEY_SIZE +
1938                                       CTR_RFC3686_NONCE_SIZE,
1939                        .max_keysize = AES_MAX_KEY_SIZE +
1940                                       CTR_RFC3686_NONCE_SIZE,
1941                        .ivsize = CTR_RFC3686_IV_SIZE,
1942                        .chunksize = AES_BLOCK_SIZE,
1943                },
1944                .caam = {
1945                        .class1_alg_type = OP_ALG_ALGSEL_AES |
1946                                           OP_ALG_AAI_CTR_MOD128,
1947                        .rfc3686 = true,
1948                },
1949        },
1950        {
1951                .skcipher = {
1952                        .base = {
1953                                .cra_name = "xts(aes)",
1954                                .cra_driver_name = "xts-aes-caam",
1955                                .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1956                                .cra_blocksize = AES_BLOCK_SIZE,
1957                        },
1958                        .setkey = xts_skcipher_setkey,
1959                        .encrypt = skcipher_encrypt,
1960                        .decrypt = skcipher_decrypt,
1961                        .min_keysize = 2 * AES_MIN_KEY_SIZE,
1962                        .max_keysize = 2 * AES_MAX_KEY_SIZE,
1963                        .ivsize = AES_BLOCK_SIZE,
1964                },
1965                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1966        },
1967        {
1968                .skcipher = {
1969                        .base = {
1970                                .cra_name = "ecb(des)",
1971                                .cra_driver_name = "ecb-des-caam",
1972                                .cra_blocksize = DES_BLOCK_SIZE,
1973                        },
1974                        .setkey = des_skcipher_setkey,
1975                        .encrypt = skcipher_encrypt,
1976                        .decrypt = skcipher_decrypt,
1977                        .min_keysize = DES_KEY_SIZE,
1978                        .max_keysize = DES_KEY_SIZE,
1979                },
1980                .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB,
1981        },
1982        {
1983                .skcipher = {
1984                        .base = {
1985                                .cra_name = "ecb(aes)",
1986                                .cra_driver_name = "ecb-aes-caam",
1987                                .cra_blocksize = AES_BLOCK_SIZE,
1988                        },
1989                        .setkey = aes_skcipher_setkey,
1990                        .encrypt = skcipher_encrypt,
1991                        .decrypt = skcipher_decrypt,
1992                        .min_keysize = AES_MIN_KEY_SIZE,
1993                        .max_keysize = AES_MAX_KEY_SIZE,
1994                },
1995                .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB,
1996        },
1997        {
1998                .skcipher = {
1999                        .base = {
2000                                .cra_name = "ecb(des3_ede)",
2001                                .cra_driver_name = "ecb-des3-caam",
2002                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2003                        },
2004                        .setkey = des3_skcipher_setkey,
2005                        .encrypt = skcipher_encrypt,
2006                        .decrypt = skcipher_decrypt,
2007                        .min_keysize = DES3_EDE_KEY_SIZE,
2008                        .max_keysize = DES3_EDE_KEY_SIZE,
2009                },
2010                .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB,
2011        },
2012};
2013
2014static struct caam_aead_alg driver_aeads[] = {
2015        {
2016                .aead = {
2017                        .base = {
2018                                .cra_name = "rfc4106(gcm(aes))",
2019                                .cra_driver_name = "rfc4106-gcm-aes-caam",
2020                                .cra_blocksize = 1,
2021                        },
2022                        .setkey = rfc4106_setkey,
2023                        .setauthsize = rfc4106_setauthsize,
2024                        .encrypt = ipsec_gcm_encrypt,
2025                        .decrypt = ipsec_gcm_decrypt,
2026                        .ivsize = GCM_RFC4106_IV_SIZE,
2027                        .maxauthsize = AES_BLOCK_SIZE,
2028                },
2029                .caam = {
2030                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2031                        .nodkp = true,
2032                },
2033        },
2034        {
2035                .aead = {
2036                        .base = {
2037                                .cra_name = "rfc4543(gcm(aes))",
2038                                .cra_driver_name = "rfc4543-gcm-aes-caam",
2039                                .cra_blocksize = 1,
2040                        },
2041                        .setkey = rfc4543_setkey,
2042                        .setauthsize = rfc4543_setauthsize,
2043                        .encrypt = ipsec_gcm_encrypt,
2044                        .decrypt = ipsec_gcm_decrypt,
2045                        .ivsize = GCM_RFC4543_IV_SIZE,
2046                        .maxauthsize = AES_BLOCK_SIZE,
2047                },
2048                .caam = {
2049                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2050                        .nodkp = true,
2051                },
2052        },
2053        /* Galois Counter Mode */
2054        {
2055                .aead = {
2056                        .base = {
2057                                .cra_name = "gcm(aes)",
2058                                .cra_driver_name = "gcm-aes-caam",
2059                                .cra_blocksize = 1,
2060                        },
2061                        .setkey = gcm_setkey,
2062                        .setauthsize = gcm_setauthsize,
2063                        .encrypt = gcm_encrypt,
2064                        .decrypt = gcm_decrypt,
2065                        .ivsize = GCM_AES_IV_SIZE,
2066                        .maxauthsize = AES_BLOCK_SIZE,
2067                },
2068                .caam = {
2069                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2070                        .nodkp = true,
2071                },
2072        },
2073        /* single-pass ipsec_esp descriptor */
2074        {
2075                .aead = {
2076                        .base = {
2077                                .cra_name = "authenc(hmac(md5),"
2078                                            "ecb(cipher_null))",
2079                                .cra_driver_name = "authenc-hmac-md5-"
2080                                                   "ecb-cipher_null-caam",
2081                                .cra_blocksize = NULL_BLOCK_SIZE,
2082                        },
2083                        .setkey = aead_setkey,
2084                        .setauthsize = aead_setauthsize,
2085                        .encrypt = aead_encrypt,
2086                        .decrypt = aead_decrypt,
2087                        .ivsize = NULL_IV_SIZE,
2088                        .maxauthsize = MD5_DIGEST_SIZE,
2089                },
2090                .caam = {
2091                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2092                                           OP_ALG_AAI_HMAC_PRECOMP,
2093                },
2094        },
2095        {
2096                .aead = {
2097                        .base = {
2098                                .cra_name = "authenc(hmac(sha1),"
2099                                            "ecb(cipher_null))",
2100                                .cra_driver_name = "authenc-hmac-sha1-"
2101                                                   "ecb-cipher_null-caam",
2102                                .cra_blocksize = NULL_BLOCK_SIZE,
2103                        },
2104                        .setkey = aead_setkey,
2105                        .setauthsize = aead_setauthsize,
2106                        .encrypt = aead_encrypt,
2107                        .decrypt = aead_decrypt,
2108                        .ivsize = NULL_IV_SIZE,
2109                        .maxauthsize = SHA1_DIGEST_SIZE,
2110                },
2111                .caam = {
2112                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2113                                           OP_ALG_AAI_HMAC_PRECOMP,
2114                },
2115        },
2116        {
2117                .aead = {
2118                        .base = {
2119                                .cra_name = "authenc(hmac(sha224),"
2120                                            "ecb(cipher_null))",
2121                                .cra_driver_name = "authenc-hmac-sha224-"
2122                                                   "ecb-cipher_null-caam",
2123                                .cra_blocksize = NULL_BLOCK_SIZE,
2124                        },
2125                        .setkey = aead_setkey,
2126                        .setauthsize = aead_setauthsize,
2127                        .encrypt = aead_encrypt,
2128                        .decrypt = aead_decrypt,
2129                        .ivsize = NULL_IV_SIZE,
2130                        .maxauthsize = SHA224_DIGEST_SIZE,
2131                },
2132                .caam = {
2133                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2134                                           OP_ALG_AAI_HMAC_PRECOMP,
2135                },
2136        },
2137        {
2138                .aead = {
2139                        .base = {
2140                                .cra_name = "authenc(hmac(sha256),"
2141                                            "ecb(cipher_null))",
2142                                .cra_driver_name = "authenc-hmac-sha256-"
2143                                                   "ecb-cipher_null-caam",
2144                                .cra_blocksize = NULL_BLOCK_SIZE,
2145                        },
2146                        .setkey = aead_setkey,
2147                        .setauthsize = aead_setauthsize,
2148                        .encrypt = aead_encrypt,
2149                        .decrypt = aead_decrypt,
2150                        .ivsize = NULL_IV_SIZE,
2151                        .maxauthsize = SHA256_DIGEST_SIZE,
2152                },
2153                .caam = {
2154                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2155                                           OP_ALG_AAI_HMAC_PRECOMP,
2156                },
2157        },
2158        {
2159                .aead = {
2160                        .base = {
2161                                .cra_name = "authenc(hmac(sha384),"
2162                                            "ecb(cipher_null))",
2163                                .cra_driver_name = "authenc-hmac-sha384-"
2164                                                   "ecb-cipher_null-caam",
2165                                .cra_blocksize = NULL_BLOCK_SIZE,
2166                        },
2167                        .setkey = aead_setkey,
2168                        .setauthsize = aead_setauthsize,
2169                        .encrypt = aead_encrypt,
2170                        .decrypt = aead_decrypt,
2171                        .ivsize = NULL_IV_SIZE,
2172                        .maxauthsize = SHA384_DIGEST_SIZE,
2173                },
2174                .caam = {
2175                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2176                                           OP_ALG_AAI_HMAC_PRECOMP,
2177                },
2178        },
2179        {
2180                .aead = {
2181                        .base = {
2182                                .cra_name = "authenc(hmac(sha512),"
2183                                            "ecb(cipher_null))",
2184                                .cra_driver_name = "authenc-hmac-sha512-"
2185                                                   "ecb-cipher_null-caam",
2186                                .cra_blocksize = NULL_BLOCK_SIZE,
2187                        },
2188                        .setkey = aead_setkey,
2189                        .setauthsize = aead_setauthsize,
2190                        .encrypt = aead_encrypt,
2191                        .decrypt = aead_decrypt,
2192                        .ivsize = NULL_IV_SIZE,
2193                        .maxauthsize = SHA512_DIGEST_SIZE,
2194                },
2195                .caam = {
2196                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2197                                           OP_ALG_AAI_HMAC_PRECOMP,
2198                },
2199        },
2200        {
2201                .aead = {
2202                        .base = {
2203                                .cra_name = "authenc(hmac(md5),cbc(aes))",
2204                                .cra_driver_name = "authenc-hmac-md5-"
2205                                                   "cbc-aes-caam",
2206                                .cra_blocksize = AES_BLOCK_SIZE,
2207                        },
2208                        .setkey = aead_setkey,
2209                        .setauthsize = aead_setauthsize,
2210                        .encrypt = aead_encrypt,
2211                        .decrypt = aead_decrypt,
2212                        .ivsize = AES_BLOCK_SIZE,
2213                        .maxauthsize = MD5_DIGEST_SIZE,
2214                },
2215                .caam = {
2216                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2217                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2218                                           OP_ALG_AAI_HMAC_PRECOMP,
2219                },
2220        },
2221        {
2222                .aead = {
2223                        .base = {
2224                                .cra_name = "echainiv(authenc(hmac(md5),"
2225                                            "cbc(aes)))",
2226                                .cra_driver_name = "echainiv-authenc-hmac-md5-"
2227                                                   "cbc-aes-caam",
2228                                .cra_blocksize = AES_BLOCK_SIZE,
2229                        },
2230                        .setkey = aead_setkey,
2231                        .setauthsize = aead_setauthsize,
2232                        .encrypt = aead_encrypt,
2233                        .decrypt = aead_decrypt,
2234                        .ivsize = AES_BLOCK_SIZE,
2235                        .maxauthsize = MD5_DIGEST_SIZE,
2236                },
2237                .caam = {
2238                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2239                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2240                                           OP_ALG_AAI_HMAC_PRECOMP,
2241                        .geniv = true,
2242                },
2243        },
2244        {
2245                .aead = {
2246                        .base = {
2247                                .cra_name = "authenc(hmac(sha1),cbc(aes))",
2248                                .cra_driver_name = "authenc-hmac-sha1-"
2249                                                   "cbc-aes-caam",
2250                                .cra_blocksize = AES_BLOCK_SIZE,
2251                        },
2252                        .setkey = aead_setkey,
2253                        .setauthsize = aead_setauthsize,
2254                        .encrypt = aead_encrypt,
2255                        .decrypt = aead_decrypt,
2256                        .ivsize = AES_BLOCK_SIZE,
2257                        .maxauthsize = SHA1_DIGEST_SIZE,
2258                },
2259                .caam = {
2260                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2261                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2262                                           OP_ALG_AAI_HMAC_PRECOMP,
2263                },
2264        },
2265        {
2266                .aead = {
2267                        .base = {
2268                                .cra_name = "echainiv(authenc(hmac(sha1),"
2269                                            "cbc(aes)))",
2270                                .cra_driver_name = "echainiv-authenc-"
2271                                                   "hmac-sha1-cbc-aes-caam",
2272                                .cra_blocksize = AES_BLOCK_SIZE,
2273                        },
2274                        .setkey = aead_setkey,
2275                        .setauthsize = aead_setauthsize,
2276                        .encrypt = aead_encrypt,
2277                        .decrypt = aead_decrypt,
2278                        .ivsize = AES_BLOCK_SIZE,
2279                        .maxauthsize = SHA1_DIGEST_SIZE,
2280                },
2281                .caam = {
2282                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2283                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2284                                           OP_ALG_AAI_HMAC_PRECOMP,
2285                        .geniv = true,
2286                },
2287        },
2288        {
2289                .aead = {
2290                        .base = {
2291                                .cra_name = "authenc(hmac(sha224),cbc(aes))",
2292                                .cra_driver_name = "authenc-hmac-sha224-"
2293                                                   "cbc-aes-caam",
2294                                .cra_blocksize = AES_BLOCK_SIZE,
2295                        },
2296                        .setkey = aead_setkey,
2297                        .setauthsize = aead_setauthsize,
2298                        .encrypt = aead_encrypt,
2299                        .decrypt = aead_decrypt,
2300                        .ivsize = AES_BLOCK_SIZE,
2301                        .maxauthsize = SHA224_DIGEST_SIZE,
2302                },
2303                .caam = {
2304                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2305                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2306                                           OP_ALG_AAI_HMAC_PRECOMP,
2307                },
2308        },
2309        {
2310                .aead = {
2311                        .base = {
2312                                .cra_name = "echainiv(authenc(hmac(sha224),"
2313                                            "cbc(aes)))",
2314                                .cra_driver_name = "echainiv-authenc-"
2315                                                   "hmac-sha224-cbc-aes-caam",
2316                                .cra_blocksize = AES_BLOCK_SIZE,
2317                        },
2318                        .setkey = aead_setkey,
2319                        .setauthsize = aead_setauthsize,
2320                        .encrypt = aead_encrypt,
2321                        .decrypt = aead_decrypt,
2322                        .ivsize = AES_BLOCK_SIZE,
2323                        .maxauthsize = SHA224_DIGEST_SIZE,
2324                },
2325                .caam = {
2326                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2327                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2328                                           OP_ALG_AAI_HMAC_PRECOMP,
2329                        .geniv = true,
2330                },
2331        },
2332        {
2333                .aead = {
2334                        .base = {
2335                                .cra_name = "authenc(hmac(sha256),cbc(aes))",
2336                                .cra_driver_name = "authenc-hmac-sha256-"
2337                                                   "cbc-aes-caam",
2338                                .cra_blocksize = AES_BLOCK_SIZE,
2339                        },
2340                        .setkey = aead_setkey,
2341                        .setauthsize = aead_setauthsize,
2342                        .encrypt = aead_encrypt,
2343                        .decrypt = aead_decrypt,
2344                        .ivsize = AES_BLOCK_SIZE,
2345                        .maxauthsize = SHA256_DIGEST_SIZE,
2346                },
2347                .caam = {
2348                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2349                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2350                                           OP_ALG_AAI_HMAC_PRECOMP,
2351                },
2352        },
2353        {
2354                .aead = {
2355                        .base = {
2356                                .cra_name = "echainiv(authenc(hmac(sha256),"
2357                                            "cbc(aes)))",
2358                                .cra_driver_name = "echainiv-authenc-"
2359                                                   "hmac-sha256-cbc-aes-caam",
2360                                .cra_blocksize = AES_BLOCK_SIZE,
2361                        },
2362                        .setkey = aead_setkey,
2363                        .setauthsize = aead_setauthsize,
2364                        .encrypt = aead_encrypt,
2365                        .decrypt = aead_decrypt,
2366                        .ivsize = AES_BLOCK_SIZE,
2367                        .maxauthsize = SHA256_DIGEST_SIZE,
2368                },
2369                .caam = {
2370                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2371                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2372                                           OP_ALG_AAI_HMAC_PRECOMP,
2373                        .geniv = true,
2374                },
2375        },
2376        {
2377                .aead = {
2378                        .base = {
2379                                .cra_name = "authenc(hmac(sha384),cbc(aes))",
2380                                .cra_driver_name = "authenc-hmac-sha384-"
2381                                                   "cbc-aes-caam",
2382                                .cra_blocksize = AES_BLOCK_SIZE,
2383                        },
2384                        .setkey = aead_setkey,
2385                        .setauthsize = aead_setauthsize,
2386                        .encrypt = aead_encrypt,
2387                        .decrypt = aead_decrypt,
2388                        .ivsize = AES_BLOCK_SIZE,
2389                        .maxauthsize = SHA384_DIGEST_SIZE,
2390                },
2391                .caam = {
2392                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2393                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2394                                           OP_ALG_AAI_HMAC_PRECOMP,
2395                },
2396        },
2397        {
2398                .aead = {
2399                        .base = {
2400                                .cra_name = "echainiv(authenc(hmac(sha384),"
2401                                            "cbc(aes)))",
2402                                .cra_driver_name = "echainiv-authenc-"
2403                                                   "hmac-sha384-cbc-aes-caam",
2404                                .cra_blocksize = AES_BLOCK_SIZE,
2405                        },
2406                        .setkey = aead_setkey,
2407                        .setauthsize = aead_setauthsize,
2408                        .encrypt = aead_encrypt,
2409                        .decrypt = aead_decrypt,
2410                        .ivsize = AES_BLOCK_SIZE,
2411                        .maxauthsize = SHA384_DIGEST_SIZE,
2412                },
2413                .caam = {
2414                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2415                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2416                                           OP_ALG_AAI_HMAC_PRECOMP,
2417                        .geniv = true,
2418                },
2419        },
2420        {
2421                .aead = {
2422                        .base = {
2423                                .cra_name = "authenc(hmac(sha512),cbc(aes))",
2424                                .cra_driver_name = "authenc-hmac-sha512-"
2425                                                   "cbc-aes-caam",
2426                                .cra_blocksize = AES_BLOCK_SIZE,
2427                        },
2428                        .setkey = aead_setkey,
2429                        .setauthsize = aead_setauthsize,
2430                        .encrypt = aead_encrypt,
2431                        .decrypt = aead_decrypt,
2432                        .ivsize = AES_BLOCK_SIZE,
2433                        .maxauthsize = SHA512_DIGEST_SIZE,
2434                },
2435                .caam = {
2436                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2437                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2438                                           OP_ALG_AAI_HMAC_PRECOMP,
2439                },
2440        },
2441        {
2442                .aead = {
2443                        .base = {
2444                                .cra_name = "echainiv(authenc(hmac(sha512),"
2445                                            "cbc(aes)))",
2446                                .cra_driver_name = "echainiv-authenc-"
2447                                                   "hmac-sha512-cbc-aes-caam",
2448                                .cra_blocksize = AES_BLOCK_SIZE,
2449                        },
2450                        .setkey = aead_setkey,
2451                        .setauthsize = aead_setauthsize,
2452                        .encrypt = aead_encrypt,
2453                        .decrypt = aead_decrypt,
2454                        .ivsize = AES_BLOCK_SIZE,
2455                        .maxauthsize = SHA512_DIGEST_SIZE,
2456                },
2457                .caam = {
2458                        .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2459                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2460                                           OP_ALG_AAI_HMAC_PRECOMP,
2461                        .geniv = true,
2462                },
2463        },
2464        {
2465                .aead = {
2466                        .base = {
2467                                .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2468                                .cra_driver_name = "authenc-hmac-md5-"
2469                                                   "cbc-des3_ede-caam",
2470                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2471                        },
2472                        .setkey = des3_aead_setkey,
2473                        .setauthsize = aead_setauthsize,
2474                        .encrypt = aead_encrypt,
2475                        .decrypt = aead_decrypt,
2476                        .ivsize = DES3_EDE_BLOCK_SIZE,
2477                        .maxauthsize = MD5_DIGEST_SIZE,
2478                },
2479                .caam = {
2480                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2481                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2482                                           OP_ALG_AAI_HMAC_PRECOMP,
2483                }
2484        },
2485        {
2486                .aead = {
2487                        .base = {
2488                                .cra_name = "echainiv(authenc(hmac(md5),"
2489                                            "cbc(des3_ede)))",
2490                                .cra_driver_name = "echainiv-authenc-hmac-md5-"
2491                                                   "cbc-des3_ede-caam",
2492                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2493                        },
2494                        .setkey = des3_aead_setkey,
2495                        .setauthsize = aead_setauthsize,
2496                        .encrypt = aead_encrypt,
2497                        .decrypt = aead_decrypt,
2498                        .ivsize = DES3_EDE_BLOCK_SIZE,
2499                        .maxauthsize = MD5_DIGEST_SIZE,
2500                },
2501                .caam = {
2502                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2503                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2504                                           OP_ALG_AAI_HMAC_PRECOMP,
2505                        .geniv = true,
2506                }
2507        },
2508        {
2509                .aead = {
2510                        .base = {
2511                                .cra_name = "authenc(hmac(sha1),"
2512                                            "cbc(des3_ede))",
2513                                .cra_driver_name = "authenc-hmac-sha1-"
2514                                                   "cbc-des3_ede-caam",
2515                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2516                        },
2517                        .setkey = des3_aead_setkey,
2518                        .setauthsize = aead_setauthsize,
2519                        .encrypt = aead_encrypt,
2520                        .decrypt = aead_decrypt,
2521                        .ivsize = DES3_EDE_BLOCK_SIZE,
2522                        .maxauthsize = SHA1_DIGEST_SIZE,
2523                },
2524                .caam = {
2525                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2526                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2527                                           OP_ALG_AAI_HMAC_PRECOMP,
2528                },
2529        },
2530        {
2531                .aead = {
2532                        .base = {
2533                                .cra_name = "echainiv(authenc(hmac(sha1),"
2534                                            "cbc(des3_ede)))",
2535                                .cra_driver_name = "echainiv-authenc-"
2536                                                   "hmac-sha1-"
2537                                                   "cbc-des3_ede-caam",
2538                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2539                        },
2540                        .setkey = des3_aead_setkey,
2541                        .setauthsize = aead_setauthsize,
2542                        .encrypt = aead_encrypt,
2543                        .decrypt = aead_decrypt,
2544                        .ivsize = DES3_EDE_BLOCK_SIZE,
2545                        .maxauthsize = SHA1_DIGEST_SIZE,
2546                },
2547                .caam = {
2548                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2549                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2550                                           OP_ALG_AAI_HMAC_PRECOMP,
2551                        .geniv = true,
2552                },
2553        },
2554        {
2555                .aead = {
2556                        .base = {
2557                                .cra_name = "authenc(hmac(sha224),"
2558                                            "cbc(des3_ede))",
2559                                .cra_driver_name = "authenc-hmac-sha224-"
2560                                                   "cbc-des3_ede-caam",
2561                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2562                        },
2563                        .setkey = des3_aead_setkey,
2564                        .setauthsize = aead_setauthsize,
2565                        .encrypt = aead_encrypt,
2566                        .decrypt = aead_decrypt,
2567                        .ivsize = DES3_EDE_BLOCK_SIZE,
2568                        .maxauthsize = SHA224_DIGEST_SIZE,
2569                },
2570                .caam = {
2571                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2572                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2573                                           OP_ALG_AAI_HMAC_PRECOMP,
2574                },
2575        },
2576        {
2577                .aead = {
2578                        .base = {
2579                                .cra_name = "echainiv(authenc(hmac(sha224),"
2580                                            "cbc(des3_ede)))",
2581                                .cra_driver_name = "echainiv-authenc-"
2582                                                   "hmac-sha224-"
2583                                                   "cbc-des3_ede-caam",
2584                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2585                        },
2586                        .setkey = des3_aead_setkey,
2587                        .setauthsize = aead_setauthsize,
2588                        .encrypt = aead_encrypt,
2589                        .decrypt = aead_decrypt,
2590                        .ivsize = DES3_EDE_BLOCK_SIZE,
2591                        .maxauthsize = SHA224_DIGEST_SIZE,
2592                },
2593                .caam = {
2594                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2595                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2596                                           OP_ALG_AAI_HMAC_PRECOMP,
2597                        .geniv = true,
2598                },
2599        },
2600        {
2601                .aead = {
2602                        .base = {
2603                                .cra_name = "authenc(hmac(sha256),"
2604                                            "cbc(des3_ede))",
2605                                .cra_driver_name = "authenc-hmac-sha256-"
2606                                                   "cbc-des3_ede-caam",
2607                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2608                        },
2609                        .setkey = des3_aead_setkey,
2610                        .setauthsize = aead_setauthsize,
2611                        .encrypt = aead_encrypt,
2612                        .decrypt = aead_decrypt,
2613                        .ivsize = DES3_EDE_BLOCK_SIZE,
2614                        .maxauthsize = SHA256_DIGEST_SIZE,
2615                },
2616                .caam = {
2617                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2618                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2619                                           OP_ALG_AAI_HMAC_PRECOMP,
2620                },
2621        },
2622        {
2623                .aead = {
2624                        .base = {
2625                                .cra_name = "echainiv(authenc(hmac(sha256),"
2626                                            "cbc(des3_ede)))",
2627                                .cra_driver_name = "echainiv-authenc-"
2628                                                   "hmac-sha256-"
2629                                                   "cbc-des3_ede-caam",
2630                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2631                        },
2632                        .setkey = des3_aead_setkey,
2633                        .setauthsize = aead_setauthsize,
2634                        .encrypt = aead_encrypt,
2635                        .decrypt = aead_decrypt,
2636                        .ivsize = DES3_EDE_BLOCK_SIZE,
2637                        .maxauthsize = SHA256_DIGEST_SIZE,
2638                },
2639                .caam = {
2640                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2641                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2642                                           OP_ALG_AAI_HMAC_PRECOMP,
2643                        .geniv = true,
2644                },
2645        },
2646        {
2647                .aead = {
2648                        .base = {
2649                                .cra_name = "authenc(hmac(sha384),"
2650                                            "cbc(des3_ede))",
2651                                .cra_driver_name = "authenc-hmac-sha384-"
2652                                                   "cbc-des3_ede-caam",
2653                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2654                        },
2655                        .setkey = des3_aead_setkey,
2656                        .setauthsize = aead_setauthsize,
2657                        .encrypt = aead_encrypt,
2658                        .decrypt = aead_decrypt,
2659                        .ivsize = DES3_EDE_BLOCK_SIZE,
2660                        .maxauthsize = SHA384_DIGEST_SIZE,
2661                },
2662                .caam = {
2663                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2664                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2665                                           OP_ALG_AAI_HMAC_PRECOMP,
2666                },
2667        },
2668        {
2669                .aead = {
2670                        .base = {
2671                                .cra_name = "echainiv(authenc(hmac(sha384),"
2672                                            "cbc(des3_ede)))",
2673                                .cra_driver_name = "echainiv-authenc-"
2674                                                   "hmac-sha384-"
2675                                                   "cbc-des3_ede-caam",
2676                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2677                        },
2678                        .setkey = des3_aead_setkey,
2679                        .setauthsize = aead_setauthsize,
2680                        .encrypt = aead_encrypt,
2681                        .decrypt = aead_decrypt,
2682                        .ivsize = DES3_EDE_BLOCK_SIZE,
2683                        .maxauthsize = SHA384_DIGEST_SIZE,
2684                },
2685                .caam = {
2686                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2687                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2688                                           OP_ALG_AAI_HMAC_PRECOMP,
2689                        .geniv = true,
2690                },
2691        },
2692        {
2693                .aead = {
2694                        .base = {
2695                                .cra_name = "authenc(hmac(sha512),"
2696                                            "cbc(des3_ede))",
2697                                .cra_driver_name = "authenc-hmac-sha512-"
2698                                                   "cbc-des3_ede-caam",
2699                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2700                        },
2701                        .setkey = des3_aead_setkey,
2702                        .setauthsize = aead_setauthsize,
2703                        .encrypt = aead_encrypt,
2704                        .decrypt = aead_decrypt,
2705                        .ivsize = DES3_EDE_BLOCK_SIZE,
2706                        .maxauthsize = SHA512_DIGEST_SIZE,
2707                },
2708                .caam = {
2709                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2710                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2711                                           OP_ALG_AAI_HMAC_PRECOMP,
2712                },
2713        },
2714        {
2715                .aead = {
2716                        .base = {
2717                                .cra_name = "echainiv(authenc(hmac(sha512),"
2718                                            "cbc(des3_ede)))",
2719                                .cra_driver_name = "echainiv-authenc-"
2720                                                   "hmac-sha512-"
2721                                                   "cbc-des3_ede-caam",
2722                                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2723                        },
2724                        .setkey = des3_aead_setkey,
2725                        .setauthsize = aead_setauthsize,
2726                        .encrypt = aead_encrypt,
2727                        .decrypt = aead_decrypt,
2728                        .ivsize = DES3_EDE_BLOCK_SIZE,
2729                        .maxauthsize = SHA512_DIGEST_SIZE,
2730                },
2731                .caam = {
2732                        .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2733                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2734                                           OP_ALG_AAI_HMAC_PRECOMP,
2735                        .geniv = true,
2736                },
2737        },
2738        {
2739                .aead = {
2740                        .base = {
2741                                .cra_name = "authenc(hmac(md5),cbc(des))",
2742                                .cra_driver_name = "authenc-hmac-md5-"
2743                                                   "cbc-des-caam",
2744                                .cra_blocksize = DES_BLOCK_SIZE,
2745                        },
2746                        .setkey = aead_setkey,
2747                        .setauthsize = aead_setauthsize,
2748                        .encrypt = aead_encrypt,
2749                        .decrypt = aead_decrypt,
2750                        .ivsize = DES_BLOCK_SIZE,
2751                        .maxauthsize = MD5_DIGEST_SIZE,
2752                },
2753                .caam = {
2754                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2755                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2756                                           OP_ALG_AAI_HMAC_PRECOMP,
2757                },
2758        },
2759        {
2760                .aead = {
2761                        .base = {
2762                                .cra_name = "echainiv(authenc(hmac(md5),"
2763                                            "cbc(des)))",
2764                                .cra_driver_name = "echainiv-authenc-hmac-md5-"
2765                                                   "cbc-des-caam",
2766                                .cra_blocksize = DES_BLOCK_SIZE,
2767                        },
2768                        .setkey = aead_setkey,
2769                        .setauthsize = aead_setauthsize,
2770                        .encrypt = aead_encrypt,
2771                        .decrypt = aead_decrypt,
2772                        .ivsize = DES_BLOCK_SIZE,
2773                        .maxauthsize = MD5_DIGEST_SIZE,
2774                },
2775                .caam = {
2776                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2777                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2778                                           OP_ALG_AAI_HMAC_PRECOMP,
2779                        .geniv = true,
2780                },
2781        },
2782        {
2783                .aead = {
2784                        .base = {
2785                                .cra_name = "authenc(hmac(sha1),cbc(des))",
2786                                .cra_driver_name = "authenc-hmac-sha1-"
2787                                                   "cbc-des-caam",
2788                                .cra_blocksize = DES_BLOCK_SIZE,
2789                        },
2790                        .setkey = aead_setkey,
2791                        .setauthsize = aead_setauthsize,
2792                        .encrypt = aead_encrypt,
2793                        .decrypt = aead_decrypt,
2794                        .ivsize = DES_BLOCK_SIZE,
2795                        .maxauthsize = SHA1_DIGEST_SIZE,
2796                },
2797                .caam = {
2798                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2799                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2800                                           OP_ALG_AAI_HMAC_PRECOMP,
2801                },
2802        },
2803        {
2804                .aead = {
2805                        .base = {
2806                                .cra_name = "echainiv(authenc(hmac(sha1),"
2807                                            "cbc(des)))",
2808                                .cra_driver_name = "echainiv-authenc-"
2809                                                   "hmac-sha1-cbc-des-caam",
2810                                .cra_blocksize = DES_BLOCK_SIZE,
2811                        },
2812                        .setkey = aead_setkey,
2813                        .setauthsize = aead_setauthsize,
2814                        .encrypt = aead_encrypt,
2815                        .decrypt = aead_decrypt,
2816                        .ivsize = DES_BLOCK_SIZE,
2817                        .maxauthsize = SHA1_DIGEST_SIZE,
2818                },
2819                .caam = {
2820                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2821                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2822                                           OP_ALG_AAI_HMAC_PRECOMP,
2823                        .geniv = true,
2824                },
2825        },
2826        {
2827                .aead = {
2828                        .base = {
2829                                .cra_name = "authenc(hmac(sha224),cbc(des))",
2830                                .cra_driver_name = "authenc-hmac-sha224-"
2831                                                   "cbc-des-caam",
2832                                .cra_blocksize = DES_BLOCK_SIZE,
2833                        },
2834                        .setkey = aead_setkey,
2835                        .setauthsize = aead_setauthsize,
2836                        .encrypt = aead_encrypt,
2837                        .decrypt = aead_decrypt,
2838                        .ivsize = DES_BLOCK_SIZE,
2839                        .maxauthsize = SHA224_DIGEST_SIZE,
2840                },
2841                .caam = {
2842                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2843                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2844                                           OP_ALG_AAI_HMAC_PRECOMP,
2845                },
2846        },
2847        {
2848                .aead = {
2849                        .base = {
2850                                .cra_name = "echainiv(authenc(hmac(sha224),"
2851                                            "cbc(des)))",
2852                                .cra_driver_name = "echainiv-authenc-"
2853                                                   "hmac-sha224-cbc-des-caam",
2854                                .cra_blocksize = DES_BLOCK_SIZE,
2855                        },
2856                        .setkey = aead_setkey,
2857                        .setauthsize = aead_setauthsize,
2858                        .encrypt = aead_encrypt,
2859                        .decrypt = aead_decrypt,
2860                        .ivsize = DES_BLOCK_SIZE,
2861                        .maxauthsize = SHA224_DIGEST_SIZE,
2862                },
2863                .caam = {
2864                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2865                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2866                                           OP_ALG_AAI_HMAC_PRECOMP,
2867                        .geniv = true,
2868                },
2869        },
2870        {
2871                .aead = {
2872                        .base = {
2873                                .cra_name = "authenc(hmac(sha256),cbc(des))",
2874                                .cra_driver_name = "authenc-hmac-sha256-"
2875                                                   "cbc-des-caam",
2876                                .cra_blocksize = DES_BLOCK_SIZE,
2877                        },
2878                        .setkey = aead_setkey,
2879                        .setauthsize = aead_setauthsize,
2880                        .encrypt = aead_encrypt,
2881                        .decrypt = aead_decrypt,
2882                        .ivsize = DES_BLOCK_SIZE,
2883                        .maxauthsize = SHA256_DIGEST_SIZE,
2884                },
2885                .caam = {
2886                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2887                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2888                                           OP_ALG_AAI_HMAC_PRECOMP,
2889                },
2890        },
2891        {
2892                .aead = {
2893                        .base = {
2894                                .cra_name = "echainiv(authenc(hmac(sha256),"
2895                                            "cbc(des)))",
2896                                .cra_driver_name = "echainiv-authenc-"
2897                                                   "hmac-sha256-cbc-des-caam",
2898                                .cra_blocksize = DES_BLOCK_SIZE,
2899                        },
2900                        .setkey = aead_setkey,
2901                        .setauthsize = aead_setauthsize,
2902                        .encrypt = aead_encrypt,
2903                        .decrypt = aead_decrypt,
2904                        .ivsize = DES_BLOCK_SIZE,
2905                        .maxauthsize = SHA256_DIGEST_SIZE,
2906                },
2907                .caam = {
2908                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2909                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2910                                           OP_ALG_AAI_HMAC_PRECOMP,
2911                        .geniv = true,
2912                },
2913        },
2914        {
2915                .aead = {
2916                        .base = {
2917                                .cra_name = "authenc(hmac(sha384),cbc(des))",
2918                                .cra_driver_name = "authenc-hmac-sha384-"
2919                                                   "cbc-des-caam",
2920                                .cra_blocksize = DES_BLOCK_SIZE,
2921                        },
2922                        .setkey = aead_setkey,
2923                        .setauthsize = aead_setauthsize,
2924                        .encrypt = aead_encrypt,
2925                        .decrypt = aead_decrypt,
2926                        .ivsize = DES_BLOCK_SIZE,
2927                        .maxauthsize = SHA384_DIGEST_SIZE,
2928                },
2929                .caam = {
2930                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2931                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2932                                           OP_ALG_AAI_HMAC_PRECOMP,
2933                },
2934        },
2935        {
2936                .aead = {
2937                        .base = {
2938                                .cra_name = "echainiv(authenc(hmac(sha384),"
2939                                            "cbc(des)))",
2940                                .cra_driver_name = "echainiv-authenc-"
2941                                                   "hmac-sha384-cbc-des-caam",
2942                                .cra_blocksize = DES_BLOCK_SIZE,
2943                        },
2944                        .setkey = aead_setkey,
2945                        .setauthsize = aead_setauthsize,
2946                        .encrypt = aead_encrypt,
2947                        .decrypt = aead_decrypt,
2948                        .ivsize = DES_BLOCK_SIZE,
2949                        .maxauthsize = SHA384_DIGEST_SIZE,
2950                },
2951                .caam = {
2952                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2953                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2954                                           OP_ALG_AAI_HMAC_PRECOMP,
2955                        .geniv = true,
2956                },
2957        },
2958        {
2959                .aead = {
2960                        .base = {
2961                                .cra_name = "authenc(hmac(sha512),cbc(des))",
2962                                .cra_driver_name = "authenc-hmac-sha512-"
2963                                                   "cbc-des-caam",
2964                                .cra_blocksize = DES_BLOCK_SIZE,
2965                        },
2966                        .setkey = aead_setkey,
2967                        .setauthsize = aead_setauthsize,
2968                        .encrypt = aead_encrypt,
2969                        .decrypt = aead_decrypt,
2970                        .ivsize = DES_BLOCK_SIZE,
2971                        .maxauthsize = SHA512_DIGEST_SIZE,
2972                },
2973                .caam = {
2974                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2975                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2976                                           OP_ALG_AAI_HMAC_PRECOMP,
2977                },
2978        },
2979        {
2980                .aead = {
2981                        .base = {
2982                                .cra_name = "echainiv(authenc(hmac(sha512),"
2983                                            "cbc(des)))",
2984                                .cra_driver_name = "echainiv-authenc-"
2985                                                   "hmac-sha512-cbc-des-caam",
2986                                .cra_blocksize = DES_BLOCK_SIZE,
2987                        },
2988                        .setkey = aead_setkey,
2989                        .setauthsize = aead_setauthsize,
2990                        .encrypt = aead_encrypt,
2991                        .decrypt = aead_decrypt,
2992                        .ivsize = DES_BLOCK_SIZE,
2993                        .maxauthsize = SHA512_DIGEST_SIZE,
2994                },
2995                .caam = {
2996                        .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2997                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2998                                           OP_ALG_AAI_HMAC_PRECOMP,
2999                        .geniv = true,
3000                },
3001        },
3002        {
3003                .aead = {
3004                        .base = {
3005                                .cra_name = "authenc(hmac(md5),"
3006                                            "rfc3686(ctr(aes)))",
3007                                .cra_driver_name = "authenc-hmac-md5-"
3008                                                   "rfc3686-ctr-aes-caam",
3009                                .cra_blocksize = 1,
3010                        },
3011                        .setkey = aead_setkey,
3012                        .setauthsize = aead_setauthsize,
3013                        .encrypt = aead_encrypt,
3014                        .decrypt = aead_decrypt,
3015                        .ivsize = CTR_RFC3686_IV_SIZE,
3016                        .maxauthsize = MD5_DIGEST_SIZE,
3017                },
3018                .caam = {
3019                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3020                                           OP_ALG_AAI_CTR_MOD128,
3021                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3022                                           OP_ALG_AAI_HMAC_PRECOMP,
3023                        .rfc3686 = true,
3024                },
3025        },
3026        {
3027                .aead = {
3028                        .base = {
3029                                .cra_name = "seqiv(authenc("
3030                                            "hmac(md5),rfc3686(ctr(aes))))",
3031                                .cra_driver_name = "seqiv-authenc-hmac-md5-"
3032                                                   "rfc3686-ctr-aes-caam",
3033                                .cra_blocksize = 1,
3034                        },
3035                        .setkey = aead_setkey,
3036                        .setauthsize = aead_setauthsize,
3037                        .encrypt = aead_encrypt,
3038                        .decrypt = aead_decrypt,
3039                        .ivsize = CTR_RFC3686_IV_SIZE,
3040                        .maxauthsize = MD5_DIGEST_SIZE,
3041                },
3042                .caam = {
3043                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3044                                           OP_ALG_AAI_CTR_MOD128,
3045                        .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3046                                           OP_ALG_AAI_HMAC_PRECOMP,
3047                        .rfc3686 = true,
3048                        .geniv = true,
3049                },
3050        },
3051        {
3052                .aead = {
3053                        .base = {
3054                                .cra_name = "authenc(hmac(sha1),"
3055                                            "rfc3686(ctr(aes)))",
3056                                .cra_driver_name = "authenc-hmac-sha1-"
3057                                                   "rfc3686-ctr-aes-caam",
3058                                .cra_blocksize = 1,
3059                        },
3060                        .setkey = aead_setkey,
3061                        .setauthsize = aead_setauthsize,
3062                        .encrypt = aead_encrypt,
3063                        .decrypt = aead_decrypt,
3064                        .ivsize = CTR_RFC3686_IV_SIZE,
3065                        .maxauthsize = SHA1_DIGEST_SIZE,
3066                },
3067                .caam = {
3068                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3069                                           OP_ALG_AAI_CTR_MOD128,
3070                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3071                                           OP_ALG_AAI_HMAC_PRECOMP,
3072                        .rfc3686 = true,
3073                },
3074        },
3075        {
3076                .aead = {
3077                        .base = {
3078                                .cra_name = "seqiv(authenc("
3079                                            "hmac(sha1),rfc3686(ctr(aes))))",
3080                                .cra_driver_name = "seqiv-authenc-hmac-sha1-"
3081                                                   "rfc3686-ctr-aes-caam",
3082                                .cra_blocksize = 1,
3083                        },
3084                        .setkey = aead_setkey,
3085                        .setauthsize = aead_setauthsize,
3086                        .encrypt = aead_encrypt,
3087                        .decrypt = aead_decrypt,
3088                        .ivsize = CTR_RFC3686_IV_SIZE,
3089                        .maxauthsize = SHA1_DIGEST_SIZE,
3090                },
3091                .caam = {
3092                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3093                                           OP_ALG_AAI_CTR_MOD128,
3094                        .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3095                                           OP_ALG_AAI_HMAC_PRECOMP,
3096                        .rfc3686 = true,
3097                        .geniv = true,
3098                },
3099        },
3100        {
3101                .aead = {
3102                        .base = {
3103                                .cra_name = "authenc(hmac(sha224),"
3104                                            "rfc3686(ctr(aes)))",
3105                                .cra_driver_name = "authenc-hmac-sha224-"
3106                                                   "rfc3686-ctr-aes-caam",
3107                                .cra_blocksize = 1,
3108                        },
3109                        .setkey = aead_setkey,
3110                        .setauthsize = aead_setauthsize,
3111                        .encrypt = aead_encrypt,
3112                        .decrypt = aead_decrypt,
3113                        .ivsize = CTR_RFC3686_IV_SIZE,
3114                        .maxauthsize = SHA224_DIGEST_SIZE,
3115                },
3116                .caam = {
3117                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3118                                           OP_ALG_AAI_CTR_MOD128,
3119                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3120                                           OP_ALG_AAI_HMAC_PRECOMP,
3121                        .rfc3686 = true,
3122                },
3123        },
3124        {
3125                .aead = {
3126                        .base = {
3127                                .cra_name = "seqiv(authenc("
3128                                            "hmac(sha224),rfc3686(ctr(aes))))",
3129                                .cra_driver_name = "seqiv-authenc-hmac-sha224-"
3130                                                   "rfc3686-ctr-aes-caam",
3131                                .cra_blocksize = 1,
3132                        },
3133                        .setkey = aead_setkey,
3134                        .setauthsize = aead_setauthsize,
3135                        .encrypt = aead_encrypt,
3136                        .decrypt = aead_decrypt,
3137                        .ivsize = CTR_RFC3686_IV_SIZE,
3138                        .maxauthsize = SHA224_DIGEST_SIZE,
3139                },
3140                .caam = {
3141                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3142                                           OP_ALG_AAI_CTR_MOD128,
3143                        .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3144                                           OP_ALG_AAI_HMAC_PRECOMP,
3145                        .rfc3686 = true,
3146                        .geniv = true,
3147                },
3148        },
3149        {
3150                .aead = {
3151                        .base = {
3152                                .cra_name = "authenc(hmac(sha256),"
3153                                            "rfc3686(ctr(aes)))",
3154                                .cra_driver_name = "authenc-hmac-sha256-"
3155                                                   "rfc3686-ctr-aes-caam",
3156                                .cra_blocksize = 1,
3157                        },
3158                        .setkey = aead_setkey,
3159                        .setauthsize = aead_setauthsize,
3160                        .encrypt = aead_encrypt,
3161                        .decrypt = aead_decrypt,
3162                        .ivsize = CTR_RFC3686_IV_SIZE,
3163                        .maxauthsize = SHA256_DIGEST_SIZE,
3164                },
3165                .caam = {
3166                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3167                                           OP_ALG_AAI_CTR_MOD128,
3168                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3169                                           OP_ALG_AAI_HMAC_PRECOMP,
3170                        .rfc3686 = true,
3171                },
3172        },
3173        {
3174                .aead = {
3175                        .base = {
3176                                .cra_name = "seqiv(authenc(hmac(sha256),"
3177                                            "rfc3686(ctr(aes))))",
3178                                .cra_driver_name = "seqiv-authenc-hmac-sha256-"
3179                                                   "rfc3686-ctr-aes-caam",
3180                                .cra_blocksize = 1,
3181                        },
3182                        .setkey = aead_setkey,
3183                        .setauthsize = aead_setauthsize,
3184                        .encrypt = aead_encrypt,
3185                        .decrypt = aead_decrypt,
3186                        .ivsize = CTR_RFC3686_IV_SIZE,
3187                        .maxauthsize = SHA256_DIGEST_SIZE,
3188                },
3189                .caam = {
3190                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3191                                           OP_ALG_AAI_CTR_MOD128,
3192                        .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3193                                           OP_ALG_AAI_HMAC_PRECOMP,
3194                        .rfc3686 = true,
3195                        .geniv = true,
3196                },
3197        },
3198        {
3199                .aead = {
3200                        .base = {
3201                                .cra_name = "authenc(hmac(sha384),"
3202                                            "rfc3686(ctr(aes)))",
3203                                .cra_driver_name = "authenc-hmac-sha384-"
3204                                                   "rfc3686-ctr-aes-caam",
3205                                .cra_blocksize = 1,
3206                        },
3207                        .setkey = aead_setkey,
3208                        .setauthsize = aead_setauthsize,
3209                        .encrypt = aead_encrypt,
3210                        .decrypt = aead_decrypt,
3211                        .ivsize = CTR_RFC3686_IV_SIZE,
3212                        .maxauthsize = SHA384_DIGEST_SIZE,
3213                },
3214                .caam = {
3215                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3216                                           OP_ALG_AAI_CTR_MOD128,
3217                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3218                                           OP_ALG_AAI_HMAC_PRECOMP,
3219                        .rfc3686 = true,
3220                },
3221        },
3222        {
3223                .aead = {
3224                        .base = {
3225                                .cra_name = "seqiv(authenc(hmac(sha384),"
3226                                            "rfc3686(ctr(aes))))",
3227                                .cra_driver_name = "seqiv-authenc-hmac-sha384-"
3228                                                   "rfc3686-ctr-aes-caam",
3229                                .cra_blocksize = 1,
3230                        },
3231                        .setkey = aead_setkey,
3232                        .setauthsize = aead_setauthsize,
3233                        .encrypt = aead_encrypt,
3234                        .decrypt = aead_decrypt,
3235                        .ivsize = CTR_RFC3686_IV_SIZE,
3236                        .maxauthsize = SHA384_DIGEST_SIZE,
3237                },
3238                .caam = {
3239                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3240                                           OP_ALG_AAI_CTR_MOD128,
3241                        .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3242                                           OP_ALG_AAI_HMAC_PRECOMP,
3243                        .rfc3686 = true,
3244                        .geniv = true,
3245                },
3246        },
3247        {
3248                .aead = {
3249                        .base = {
3250                                .cra_name = "authenc(hmac(sha512),"
3251                                            "rfc3686(ctr(aes)))",
3252                                .cra_driver_name = "authenc-hmac-sha512-"
3253                                                   "rfc3686-ctr-aes-caam",
3254                                .cra_blocksize = 1,
3255                        },
3256                        .setkey = aead_setkey,
3257                        .setauthsize = aead_setauthsize,
3258                        .encrypt = aead_encrypt,
3259                        .decrypt = aead_decrypt,
3260                        .ivsize = CTR_RFC3686_IV_SIZE,
3261                        .maxauthsize = SHA512_DIGEST_SIZE,
3262                },
3263                .caam = {
3264                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3265                                           OP_ALG_AAI_CTR_MOD128,
3266                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3267                                           OP_ALG_AAI_HMAC_PRECOMP,
3268                        .rfc3686 = true,
3269                },
3270        },
3271        {
3272                .aead = {
3273                        .base = {
3274                                .cra_name = "seqiv(authenc(hmac(sha512),"
3275                                            "rfc3686(ctr(aes))))",
3276                                .cra_driver_name = "seqiv-authenc-hmac-sha512-"
3277                                                   "rfc3686-ctr-aes-caam",
3278                                .cra_blocksize = 1,
3279                        },
3280                        .setkey = aead_setkey,
3281                        .setauthsize = aead_setauthsize,
3282                        .encrypt = aead_encrypt,
3283                        .decrypt = aead_decrypt,
3284                        .ivsize = CTR_RFC3686_IV_SIZE,
3285                        .maxauthsize = SHA512_DIGEST_SIZE,
3286                },
3287                .caam = {
3288                        .class1_alg_type = OP_ALG_ALGSEL_AES |
3289                                           OP_ALG_AAI_CTR_MOD128,
3290                        .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3291                                           OP_ALG_AAI_HMAC_PRECOMP,
3292                        .rfc3686 = true,
3293                        .geniv = true,
3294                },
3295        },
3296        {
3297                .aead = {
3298                        .base = {
3299                                .cra_name = "rfc7539(chacha20,poly1305)",
3300                                .cra_driver_name = "rfc7539-chacha20-poly1305-"
3301                                                   "caam",
3302                                .cra_blocksize = 1,
3303                        },
3304                        .setkey = chachapoly_setkey,
3305                        .setauthsize = chachapoly_setauthsize,
3306                        .encrypt = chachapoly_encrypt,
3307                        .decrypt = chachapoly_decrypt,
3308                        .ivsize = CHACHAPOLY_IV_SIZE,
3309                        .maxauthsize = POLY1305_DIGEST_SIZE,
3310                },
3311                .caam = {
3312                        .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3313                                           OP_ALG_AAI_AEAD,
3314                        .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3315                                           OP_ALG_AAI_AEAD,
3316                        .nodkp = true,
3317                },
3318        },
3319        {
3320                .aead = {
3321                        .base = {
3322                                .cra_name = "rfc7539esp(chacha20,poly1305)",
3323                                .cra_driver_name = "rfc7539esp-chacha20-"
3324                                                   "poly1305-caam",
3325                                .cra_blocksize = 1,
3326                        },
3327                        .setkey = chachapoly_setkey,
3328                        .setauthsize = chachapoly_setauthsize,
3329                        .encrypt = chachapoly_encrypt,
3330                        .decrypt = chachapoly_decrypt,
3331                        .ivsize = 8,
3332                        .maxauthsize = POLY1305_DIGEST_SIZE,
3333                },
3334                .caam = {
3335                        .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3336                                           OP_ALG_AAI_AEAD,
3337                        .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3338                                           OP_ALG_AAI_AEAD,
3339                        .nodkp = true,
3340                },
3341        },
3342};
3343
3344static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
3345                            bool uses_dkp)
3346{
3347        dma_addr_t dma_addr;
3348        struct caam_drv_private *priv;
3349        const size_t sh_desc_enc_offset = offsetof(struct caam_ctx,
3350                                                   sh_desc_enc);
3351
3352        ctx->jrdev = caam_jr_alloc();
3353        if (IS_ERR(ctx->jrdev)) {
3354                pr_err("Job Ring Device allocation for transform failed\n");
3355                return PTR_ERR(ctx->jrdev);
3356        }
3357
3358        priv = dev_get_drvdata(ctx->jrdev->parent);
3359        if (priv->era >= 6 && uses_dkp)
3360                ctx->dir = DMA_BIDIRECTIONAL;
3361        else
3362                ctx->dir = DMA_TO_DEVICE;
3363
3364        dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
3365                                        offsetof(struct caam_ctx,
3366                                                 sh_desc_enc_dma) -
3367                                        sh_desc_enc_offset,
3368                                        ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3369        if (dma_mapping_error(ctx->jrdev, dma_addr)) {
3370                dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
3371                caam_jr_free(ctx->jrdev);
3372                return -ENOMEM;
3373        }
3374
3375        ctx->sh_desc_enc_dma = dma_addr;
3376        ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
3377                                                   sh_desc_dec) -
3378                                        sh_desc_enc_offset;
3379        ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) -
3380                                        sh_desc_enc_offset;
3381
3382        /* copy descriptor header template value */
3383        ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3384        ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
3385
3386        return 0;
3387}
3388
3389static int caam_cra_init(struct crypto_skcipher *tfm)
3390{
3391        struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3392        struct caam_skcipher_alg *caam_alg =
3393                container_of(alg, typeof(*caam_alg), skcipher);
3394        struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
3395        u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3396        int ret = 0;
3397
3398        ctx->enginectx.op.do_one_request = skcipher_do_one_req;
3399
3400        if (alg_aai == OP_ALG_AAI_XTS) {
3401                const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
3402                struct crypto_skcipher *fallback;
3403
3404                fallback = crypto_alloc_skcipher(tfm_name, 0,
3405                                                 CRYPTO_ALG_NEED_FALLBACK);
3406                if (IS_ERR(fallback)) {
3407                        dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n",
3408                                tfm_name, PTR_ERR(fallback));
3409                        return PTR_ERR(fallback);
3410                }
3411
3412                ctx->fallback = fallback;
3413                crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
3414                                            crypto_skcipher_reqsize(fallback));
3415        } else {
3416                crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx));
3417        }
3418
3419        ret = caam_init_common(ctx, &caam_alg->caam, false);
3420        if (ret && ctx->fallback)
3421                crypto_free_skcipher(ctx->fallback);
3422
3423        return ret;
3424}
3425
3426static int caam_aead_init(struct crypto_aead *tfm)
3427{
3428        struct aead_alg *alg = crypto_aead_alg(tfm);
3429        struct caam_aead_alg *caam_alg =
3430                 container_of(alg, struct caam_aead_alg, aead);
3431        struct caam_ctx *ctx = crypto_aead_ctx(tfm);
3432
3433        crypto_aead_set_reqsize(tfm, sizeof(struct caam_aead_req_ctx));
3434
3435        ctx->enginectx.op.do_one_request = aead_do_one_req;
3436
3437        return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
3438}
3439
3440static void caam_exit_common(struct caam_ctx *ctx)
3441{
3442        dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
3443                               offsetof(struct caam_ctx, sh_desc_enc_dma) -
3444                               offsetof(struct caam_ctx, sh_desc_enc),
3445                               ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3446        caam_jr_free(ctx->jrdev);
3447}
3448
3449static void caam_cra_exit(struct crypto_skcipher *tfm)
3450{
3451        struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
3452
3453        if (ctx->fallback)
3454                crypto_free_skcipher(ctx->fallback);
3455        caam_exit_common(ctx);
3456}
3457
3458static void caam_aead_exit(struct crypto_aead *tfm)
3459{
3460        caam_exit_common(crypto_aead_ctx(tfm));
3461}
3462
3463void caam_algapi_exit(void)
3464{
3465        int i;
3466
3467        for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3468                struct caam_aead_alg *t_alg = driver_aeads + i;
3469
3470                if (t_alg->registered)
3471                        crypto_unregister_aead(&t_alg->aead);
3472        }
3473
3474        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3475                struct caam_skcipher_alg *t_alg = driver_algs + i;
3476
3477                if (t_alg->registered)
3478                        crypto_unregister_skcipher(&t_alg->skcipher);
3479        }
3480}
3481
3482static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3483{
3484        struct skcipher_alg *alg = &t_alg->skcipher;
3485
3486        alg->base.cra_module = THIS_MODULE;
3487        alg->base.cra_priority = CAAM_CRA_PRIORITY;
3488        alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3489        alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3490                              CRYPTO_ALG_KERN_DRIVER_ONLY);
3491
3492        alg->init = caam_cra_init;
3493        alg->exit = caam_cra_exit;
3494}
3495
3496static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3497{
3498        struct aead_alg *alg = &t_alg->aead;
3499
3500        alg->base.cra_module = THIS_MODULE;
3501        alg->base.cra_priority = CAAM_CRA_PRIORITY;
3502        alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3503        alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3504                              CRYPTO_ALG_KERN_DRIVER_ONLY;
3505
3506        alg->init = caam_aead_init;
3507        alg->exit = caam_aead_exit;
3508}
3509
3510int caam_algapi_init(struct device *ctrldev)
3511{
3512        struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
3513        int i = 0, err = 0;
3514        u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
3515        unsigned int md_limit = SHA512_DIGEST_SIZE;
3516        bool registered = false, gcm_support;
3517
3518        /*
3519         * Register crypto algorithms the device supports.
3520         * First, detect presence and attributes of DES, AES, and MD blocks.
3521         */
3522        if (priv->era < 10) {
3523                u32 cha_vid, cha_inst, aes_rn;
3524
3525                cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
3526                aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
3527                md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3528
3529                cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
3530                des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
3531                           CHA_ID_LS_DES_SHIFT;
3532                aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
3533                md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3534                ccha_inst = 0;
3535                ptha_inst = 0;
3536
3537                aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) &
3538                         CHA_ID_LS_AES_MASK;
3539                gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8);
3540        } else {
3541                u32 aesa, mdha;
3542
3543                aesa = rd_reg32(&priv->ctrl->vreg.aesa);
3544                mdha = rd_reg32(&priv->ctrl->vreg.mdha);
3545
3546                aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3547                md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3548
3549                des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
3550                aes_inst = aesa & CHA_VER_NUM_MASK;
3551                md_inst = mdha & CHA_VER_NUM_MASK;
3552                ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK;
3553                ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK;
3554
3555                gcm_support = aesa & CHA_VER_MISC_AES_GCM;
3556        }
3557
3558        /* If MD is present, limit digest size based on LP256 */
3559        if (md_inst && md_vid  == CHA_VER_VID_MD_LP256)
3560                md_limit = SHA256_DIGEST_SIZE;
3561
3562        for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3563                struct caam_skcipher_alg *t_alg = driver_algs + i;
3564                u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
3565
3566                /* Skip DES algorithms if not supported by device */
3567                if (!des_inst &&
3568                    ((alg_sel == OP_ALG_ALGSEL_3DES) ||
3569                     (alg_sel == OP_ALG_ALGSEL_DES)))
3570                                continue;
3571
3572                /* Skip AES algorithms if not supported by device */
3573                if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
3574                                continue;
3575
3576                /*
3577                 * Check support for AES modes not available
3578                 * on LP devices.
3579                 */
3580                if (aes_vid == CHA_VER_VID_AES_LP &&
3581                    (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) ==
3582                    OP_ALG_AAI_XTS)
3583                        continue;
3584
3585                caam_skcipher_alg_init(t_alg);
3586
3587                err = crypto_register_skcipher(&t_alg->skcipher);
3588                if (err) {
3589                        pr_warn("%s alg registration failed\n",
3590                                t_alg->skcipher.base.cra_driver_name);
3591                        continue;
3592                }
3593
3594                t_alg->registered = true;
3595                registered = true;
3596        }
3597
3598        for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3599                struct caam_aead_alg *t_alg = driver_aeads + i;
3600                u32 c1_alg_sel = t_alg->caam.class1_alg_type &
3601                                 OP_ALG_ALGSEL_MASK;
3602                u32 c2_alg_sel = t_alg->caam.class2_alg_type &
3603                                 OP_ALG_ALGSEL_MASK;
3604                u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3605
3606                /* Skip DES algorithms if not supported by device */
3607                if (!des_inst &&
3608                    ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
3609                     (c1_alg_sel == OP_ALG_ALGSEL_DES)))
3610                                continue;
3611
3612                /* Skip AES algorithms if not supported by device */
3613                if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
3614                                continue;
3615
3616                /* Skip CHACHA20 algorithms if not supported by device */
3617                if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst)
3618                        continue;
3619
3620                /* Skip POLY1305 algorithms if not supported by device */
3621                if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst)
3622                        continue;
3623
3624                /* Skip GCM algorithms if not supported by device */
3625                if (c1_alg_sel == OP_ALG_ALGSEL_AES &&
3626                    alg_aai == OP_ALG_AAI_GCM && !gcm_support)
3627                        continue;
3628
3629                /*
3630                 * Skip algorithms requiring message digests
3631                 * if MD or MD size is not supported by device.
3632                 */
3633                if (is_mdha(c2_alg_sel) &&
3634                    (!md_inst || t_alg->aead.maxauthsize > md_limit))
3635                        continue;
3636
3637                caam_aead_alg_init(t_alg);
3638
3639                err = crypto_register_aead(&t_alg->aead);
3640                if (err) {
3641                        pr_warn("%s alg registration failed\n",
3642                                t_alg->aead.base.cra_driver_name);
3643                        continue;
3644                }
3645
3646                t_alg->registered = true;
3647                registered = true;
3648        }
3649
3650        if (registered)
3651                pr_info("caam algorithms registered in /proc/crypto\n");
3652
3653        return err;
3654}
3655