linux/crypto/testmgr.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Algorithm testing framework and tests.
   4 *
   5 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
   6 * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
   7 * Copyright (c) 2007 Nokia Siemens Networks
   8 * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
   9 * Copyright (c) 2019 Google LLC
  10 *
  11 * Updated RFC4106 AES-GCM testing.
  12 *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
  13 *             Adrian Hoban <adrian.hoban@intel.com>
  14 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  15 *             Tadeusz Struk (tadeusz.struk@intel.com)
  16 *    Copyright (c) 2010, Intel Corporation.
  17 */
  18
  19#include <crypto/aead.h>
  20#include <crypto/hash.h>
  21#include <crypto/skcipher.h>
  22#include <linux/err.h>
  23#include <linux/fips.h>
  24#include <linux/module.h>
  25#include <linux/once.h>
  26#include <linux/random.h>
  27#include <linux/scatterlist.h>
  28#include <linux/slab.h>
  29#include <linux/string.h>
  30#include <linux/uio.h>
  31#include <crypto/rng.h>
  32#include <crypto/drbg.h>
  33#include <crypto/akcipher.h>
  34#include <crypto/kpp.h>
  35#include <crypto/acompress.h>
  36#include <crypto/internal/cipher.h>
  37#include <crypto/internal/simd.h>
  38
  39#include "internal.h"
  40
  41MODULE_IMPORT_NS(CRYPTO_INTERNAL);
  42
  43static bool notests;
  44module_param(notests, bool, 0644);
  45MODULE_PARM_DESC(notests, "disable crypto self-tests");
  46
  47static bool panic_on_fail;
  48module_param(panic_on_fail, bool, 0444);
  49
  50#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
  51static bool noextratests;
  52module_param(noextratests, bool, 0644);
  53MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests");
  54
  55static unsigned int fuzz_iterations = 100;
  56module_param(fuzz_iterations, uint, 0644);
  57MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");
  58
  59DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test);
  60EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test);
  61#endif
  62
  63#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS
  64
  65/* a perfect nop */
  66int alg_test(const char *driver, const char *alg, u32 type, u32 mask)
  67{
  68        return 0;
  69}
  70
  71#else
  72
  73#include "testmgr.h"
  74
  75/*
  76 * Need slab memory for testing (size in number of pages).
  77 */
  78#define XBUFSIZE        8
  79
  80/*
  81* Used by test_cipher()
  82*/
  83#define ENCRYPT 1
  84#define DECRYPT 0
  85
  86struct aead_test_suite {
  87        const struct aead_testvec *vecs;
  88        unsigned int count;
  89
  90        /*
  91         * Set if trying to decrypt an inauthentic ciphertext with this
  92         * algorithm might result in EINVAL rather than EBADMSG, due to other
  93         * validation the algorithm does on the inputs such as length checks.
  94         */
  95        unsigned int einval_allowed : 1;
  96
  97        /*
  98         * Set if this algorithm requires that the IV be located at the end of
  99         * the AAD buffer, in addition to being given in the normal way.  The
 100         * behavior when the two IV copies differ is implementation-defined.
 101         */
 102        unsigned int aad_iv : 1;
 103};
 104
 105struct cipher_test_suite {
 106        const struct cipher_testvec *vecs;
 107        unsigned int count;
 108};
 109
 110struct comp_test_suite {
 111        struct {
 112                const struct comp_testvec *vecs;
 113                unsigned int count;
 114        } comp, decomp;
 115};
 116
 117struct hash_test_suite {
 118        const struct hash_testvec *vecs;
 119        unsigned int count;
 120};
 121
 122struct cprng_test_suite {
 123        const struct cprng_testvec *vecs;
 124        unsigned int count;
 125};
 126
 127struct drbg_test_suite {
 128        const struct drbg_testvec *vecs;
 129        unsigned int count;
 130};
 131
 132struct akcipher_test_suite {
 133        const struct akcipher_testvec *vecs;
 134        unsigned int count;
 135};
 136
 137struct kpp_test_suite {
 138        const struct kpp_testvec *vecs;
 139        unsigned int count;
 140};
 141
 142struct alg_test_desc {
 143        const char *alg;
 144        const char *generic_driver;
 145        int (*test)(const struct alg_test_desc *desc, const char *driver,
 146                    u32 type, u32 mask);
 147        int fips_allowed;       /* set if alg is allowed in fips mode */
 148
 149        union {
 150                struct aead_test_suite aead;
 151                struct cipher_test_suite cipher;
 152                struct comp_test_suite comp;
 153                struct hash_test_suite hash;
 154                struct cprng_test_suite cprng;
 155                struct drbg_test_suite drbg;
 156                struct akcipher_test_suite akcipher;
 157                struct kpp_test_suite kpp;
 158        } suite;
 159};
 160
 161static void hexdump(unsigned char *buf, unsigned int len)
 162{
 163        print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
 164                        16, 1,
 165                        buf, len, false);
 166}
 167
 168static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)
 169{
 170        int i;
 171
 172        for (i = 0; i < XBUFSIZE; i++) {
 173                buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);
 174                if (!buf[i])
 175                        goto err_free_buf;
 176        }
 177
 178        return 0;
 179
 180err_free_buf:
 181        while (i-- > 0)
 182                free_pages((unsigned long)buf[i], order);
 183
 184        return -ENOMEM;
 185}
 186
 187static int testmgr_alloc_buf(char *buf[XBUFSIZE])
 188{
 189        return __testmgr_alloc_buf(buf, 0);
 190}
 191
 192static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)
 193{
 194        int i;
 195
 196        for (i = 0; i < XBUFSIZE; i++)
 197                free_pages((unsigned long)buf[i], order);
 198}
 199
 200static void testmgr_free_buf(char *buf[XBUFSIZE])
 201{
 202        __testmgr_free_buf(buf, 0);
 203}
 204
 205#define TESTMGR_POISON_BYTE     0xfe
 206#define TESTMGR_POISON_LEN      16
 207
 208static inline void testmgr_poison(void *addr, size_t len)
 209{
 210        memset(addr, TESTMGR_POISON_BYTE, len);
 211}
 212
 213/* Is the memory region still fully poisoned? */
 214static inline bool testmgr_is_poison(const void *addr, size_t len)
 215{
 216        return memchr_inv(addr, TESTMGR_POISON_BYTE, len) == NULL;
 217}
 218
 219/* flush type for hash algorithms */
 220enum flush_type {
 221        /* merge with update of previous buffer(s) */
 222        FLUSH_TYPE_NONE = 0,
 223
 224        /* update with previous buffer(s) before doing this one */
 225        FLUSH_TYPE_FLUSH,
 226
 227        /* likewise, but also export and re-import the intermediate state */
 228        FLUSH_TYPE_REIMPORT,
 229};
 230
 231/* finalization function for hash algorithms */
 232enum finalization_type {
 233        FINALIZATION_TYPE_FINAL,        /* use final() */
 234        FINALIZATION_TYPE_FINUP,        /* use finup() */
 235        FINALIZATION_TYPE_DIGEST,       /* use digest() */
 236};
 237
 238#define TEST_SG_TOTAL   10000
 239
 240/**
 241 * struct test_sg_division - description of a scatterlist entry
 242 *
 243 * This struct describes one entry of a scatterlist being constructed to check a
 244 * crypto test vector.
 245 *
 246 * @proportion_of_total: length of this chunk relative to the total length,
 247 *                       given as a proportion out of TEST_SG_TOTAL so that it
 248 *                       scales to fit any test vector
 249 * @offset: byte offset into a 2-page buffer at which this chunk will start
 250 * @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the
 251 *                                @offset
 252 * @flush_type: for hashes, whether an update() should be done now vs.
 253 *              continuing to accumulate data
 254 * @nosimd: if doing the pending update(), do it with SIMD disabled?
 255 */
 256struct test_sg_division {
 257        unsigned int proportion_of_total;
 258        unsigned int offset;
 259        bool offset_relative_to_alignmask;
 260        enum flush_type flush_type;
 261        bool nosimd;
 262};
 263
 264/**
 265 * struct testvec_config - configuration for testing a crypto test vector
 266 *
 267 * This struct describes the data layout and other parameters with which each
 268 * crypto test vector can be tested.
 269 *
 270 * @name: name of this config, logged for debugging purposes if a test fails
 271 * @inplace: operate on the data in-place, if applicable for the algorithm type?
 272 * @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP
 273 * @src_divs: description of how to arrange the source scatterlist
 274 * @dst_divs: description of how to arrange the dst scatterlist, if applicable
 275 *            for the algorithm type.  Defaults to @src_divs if unset.
 276 * @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],
 277 *             where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary
 278 * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 279 *                                   the @iv_offset
 280 * @key_offset: misalignment of the key, where 0 is default alignment
 281 * @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to
 282 *                                    the @key_offset
 283 * @finalization_type: what finalization function to use for hashes
 284 * @nosimd: execute with SIMD disabled?  Requires !CRYPTO_TFM_REQ_MAY_SLEEP.
 285 */
 286struct testvec_config {
 287        const char *name;
 288        bool inplace;
 289        u32 req_flags;
 290        struct test_sg_division src_divs[XBUFSIZE];
 291        struct test_sg_division dst_divs[XBUFSIZE];
 292        unsigned int iv_offset;
 293        unsigned int key_offset;
 294        bool iv_offset_relative_to_alignmask;
 295        bool key_offset_relative_to_alignmask;
 296        enum finalization_type finalization_type;
 297        bool nosimd;
 298};
 299
 300#define TESTVEC_CONFIG_NAMELEN  192
 301
 302/*
 303 * The following are the lists of testvec_configs to test for each algorithm
 304 * type when the basic crypto self-tests are enabled, i.e. when
 305 * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset.  They aim to provide good test
 306 * coverage, while keeping the test time much shorter than the full fuzz tests
 307 * so that the basic tests can be enabled in a wider range of circumstances.
 308 */
 309
 310/* Configs for skciphers and aeads */
 311static const struct testvec_config default_cipher_testvec_configs[] = {
 312        {
 313                .name = "in-place",
 314                .inplace = true,
 315                .src_divs = { { .proportion_of_total = 10000 } },
 316        }, {
 317                .name = "out-of-place",
 318                .src_divs = { { .proportion_of_total = 10000 } },
 319        }, {
 320                .name = "unaligned buffer, offset=1",
 321                .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 322                .iv_offset = 1,
 323                .key_offset = 1,
 324        }, {
 325                .name = "buffer aligned only to alignmask",
 326                .src_divs = {
 327                        {
 328                                .proportion_of_total = 10000,
 329                                .offset = 1,
 330                                .offset_relative_to_alignmask = true,
 331                        },
 332                },
 333                .iv_offset = 1,
 334                .iv_offset_relative_to_alignmask = true,
 335                .key_offset = 1,
 336                .key_offset_relative_to_alignmask = true,
 337        }, {
 338                .name = "two even aligned splits",
 339                .src_divs = {
 340                        { .proportion_of_total = 5000 },
 341                        { .proportion_of_total = 5000 },
 342                },
 343        }, {
 344                .name = "uneven misaligned splits, may sleep",
 345                .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 346                .src_divs = {
 347                        { .proportion_of_total = 1900, .offset = 33 },
 348                        { .proportion_of_total = 3300, .offset = 7  },
 349                        { .proportion_of_total = 4800, .offset = 18 },
 350                },
 351                .iv_offset = 3,
 352                .key_offset = 3,
 353        }, {
 354                .name = "misaligned splits crossing pages, inplace",
 355                .inplace = true,
 356                .src_divs = {
 357                        {
 358                                .proportion_of_total = 7500,
 359                                .offset = PAGE_SIZE - 32
 360                        }, {
 361                                .proportion_of_total = 2500,
 362                                .offset = PAGE_SIZE - 7
 363                        },
 364                },
 365        }
 366};
 367
 368static const struct testvec_config default_hash_testvec_configs[] = {
 369        {
 370                .name = "init+update+final aligned buffer",
 371                .src_divs = { { .proportion_of_total = 10000 } },
 372                .finalization_type = FINALIZATION_TYPE_FINAL,
 373        }, {
 374                .name = "init+finup aligned buffer",
 375                .src_divs = { { .proportion_of_total = 10000 } },
 376                .finalization_type = FINALIZATION_TYPE_FINUP,
 377        }, {
 378                .name = "digest aligned buffer",
 379                .src_divs = { { .proportion_of_total = 10000 } },
 380                .finalization_type = FINALIZATION_TYPE_DIGEST,
 381        }, {
 382                .name = "init+update+final misaligned buffer",
 383                .src_divs = { { .proportion_of_total = 10000, .offset = 1 } },
 384                .finalization_type = FINALIZATION_TYPE_FINAL,
 385                .key_offset = 1,
 386        }, {
 387                .name = "digest buffer aligned only to alignmask",
 388                .src_divs = {
 389                        {
 390                                .proportion_of_total = 10000,
 391                                .offset = 1,
 392                                .offset_relative_to_alignmask = true,
 393                        },
 394                },
 395                .finalization_type = FINALIZATION_TYPE_DIGEST,
 396                .key_offset = 1,
 397                .key_offset_relative_to_alignmask = true,
 398        }, {
 399                .name = "init+update+update+final two even splits",
 400                .src_divs = {
 401                        { .proportion_of_total = 5000 },
 402                        {
 403                                .proportion_of_total = 5000,
 404                                .flush_type = FLUSH_TYPE_FLUSH,
 405                        },
 406                },
 407                .finalization_type = FINALIZATION_TYPE_FINAL,
 408        }, {
 409                .name = "digest uneven misaligned splits, may sleep",
 410                .req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,
 411                .src_divs = {
 412                        { .proportion_of_total = 1900, .offset = 33 },
 413                        { .proportion_of_total = 3300, .offset = 7  },
 414                        { .proportion_of_total = 4800, .offset = 18 },
 415                },
 416                .finalization_type = FINALIZATION_TYPE_DIGEST,
 417        }, {
 418                .name = "digest misaligned splits crossing pages",
 419                .src_divs = {
 420                        {
 421                                .proportion_of_total = 7500,
 422                                .offset = PAGE_SIZE - 32,
 423                        }, {
 424                                .proportion_of_total = 2500,
 425                                .offset = PAGE_SIZE - 7,
 426                        },
 427                },
 428                .finalization_type = FINALIZATION_TYPE_DIGEST,
 429        }, {
 430                .name = "import/export",
 431                .src_divs = {
 432                        {
 433                                .proportion_of_total = 6500,
 434                                .flush_type = FLUSH_TYPE_REIMPORT,
 435                        }, {
 436                                .proportion_of_total = 3500,
 437                                .flush_type = FLUSH_TYPE_REIMPORT,
 438                        },
 439                },
 440                .finalization_type = FINALIZATION_TYPE_FINAL,
 441        }
 442};
 443
 444static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)
 445{
 446        unsigned int remaining = TEST_SG_TOTAL;
 447        unsigned int ndivs = 0;
 448
 449        do {
 450                remaining -= divs[ndivs++].proportion_of_total;
 451        } while (remaining);
 452
 453        return ndivs;
 454}
 455
 456#define SGDIVS_HAVE_FLUSHES     BIT(0)
 457#define SGDIVS_HAVE_NOSIMD      BIT(1)
 458
 459static bool valid_sg_divisions(const struct test_sg_division *divs,
 460                               unsigned int count, int *flags_ret)
 461{
 462        unsigned int total = 0;
 463        unsigned int i;
 464
 465        for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {
 466                if (divs[i].proportion_of_total <= 0 ||
 467                    divs[i].proportion_of_total > TEST_SG_TOTAL - total)
 468                        return false;
 469                total += divs[i].proportion_of_total;
 470                if (divs[i].flush_type != FLUSH_TYPE_NONE)
 471                        *flags_ret |= SGDIVS_HAVE_FLUSHES;
 472                if (divs[i].nosimd)
 473                        *flags_ret |= SGDIVS_HAVE_NOSIMD;
 474        }
 475        return total == TEST_SG_TOTAL &&
 476                memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL;
 477}
 478
 479/*
 480 * Check whether the given testvec_config is valid.  This isn't strictly needed
 481 * since every testvec_config should be valid, but check anyway so that people
 482 * don't unknowingly add broken configs that don't do what they wanted.
 483 */
 484static bool valid_testvec_config(const struct testvec_config *cfg)
 485{
 486        int flags = 0;
 487
 488        if (cfg->name == NULL)
 489                return false;
 490
 491        if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs),
 492                                &flags))
 493                return false;
 494
 495        if (cfg->dst_divs[0].proportion_of_total) {
 496                if (!valid_sg_divisions(cfg->dst_divs,
 497                                        ARRAY_SIZE(cfg->dst_divs), &flags))
 498                        return false;
 499        } else {
 500                if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs)))
 501                        return false;
 502                /* defaults to dst_divs=src_divs */
 503        }
 504
 505        if (cfg->iv_offset +
 506            (cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >
 507            MAX_ALGAPI_ALIGNMASK + 1)
 508                return false;
 509
 510        if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) &&
 511            cfg->finalization_type == FINALIZATION_TYPE_DIGEST)
 512                return false;
 513
 514        if ((cfg->nosimd || (flags & SGDIVS_HAVE_NOSIMD)) &&
 515            (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))
 516                return false;
 517
 518        return true;
 519}
 520
 521struct test_sglist {
 522        char *bufs[XBUFSIZE];
 523        struct scatterlist sgl[XBUFSIZE];
 524        struct scatterlist sgl_saved[XBUFSIZE];
 525        struct scatterlist *sgl_ptr;
 526        unsigned int nents;
 527};
 528
 529static int init_test_sglist(struct test_sglist *tsgl)
 530{
 531        return __testmgr_alloc_buf(tsgl->bufs, 1 /* two pages per buffer */);
 532}
 533
 534static void destroy_test_sglist(struct test_sglist *tsgl)
 535{
 536        return __testmgr_free_buf(tsgl->bufs, 1 /* two pages per buffer */);
 537}
 538
 539/**
 540 * build_test_sglist() - build a scatterlist for a crypto test
 541 *
 542 * @tsgl: the scatterlist to build.  @tsgl->bufs[] contains an array of 2-page
 543 *        buffers which the scatterlist @tsgl->sgl[] will be made to point into.
 544 * @divs: the layout specification on which the scatterlist will be based
 545 * @alignmask: the algorithm's alignmask
 546 * @total_len: the total length of the scatterlist to build in bytes
 547 * @data: if non-NULL, the buffers will be filled with this data until it ends.
 548 *        Otherwise the buffers will be poisoned.  In both cases, some bytes
 549 *        past the end of each buffer will be poisoned to help detect overruns.
 550 * @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry
 551 *            corresponds will be returned here.  This will match @divs except
 552 *            that divisions resolving to a length of 0 are omitted as they are
 553 *            not included in the scatterlist.
 554 *
 555 * Return: 0 or a -errno value
 556 */
 557static int build_test_sglist(struct test_sglist *tsgl,
 558                             const struct test_sg_division *divs,
 559                             const unsigned int alignmask,
 560                             const unsigned int total_len,
 561                             struct iov_iter *data,
 562                             const struct test_sg_division *out_divs[XBUFSIZE])
 563{
 564        struct {
 565                const struct test_sg_division *div;
 566                size_t length;
 567        } partitions[XBUFSIZE];
 568        const unsigned int ndivs = count_test_sg_divisions(divs);
 569        unsigned int len_remaining = total_len;
 570        unsigned int i;
 571
 572        BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));
 573        if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))
 574                return -EINVAL;
 575
 576        /* Calculate the (div, length) pairs */
 577        tsgl->nents = 0;
 578        for (i = 0; i < ndivs; i++) {
 579                unsigned int len_this_sg =
 580                        min(len_remaining,
 581                            (total_len * divs[i].proportion_of_total +
 582                             TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);
 583
 584                if (len_this_sg != 0) {
 585                        partitions[tsgl->nents].div = &divs[i];
 586                        partitions[tsgl->nents].length = len_this_sg;
 587                        tsgl->nents++;
 588                        len_remaining -= len_this_sg;
 589                }
 590        }
 591        if (tsgl->nents == 0) {
 592                partitions[tsgl->nents].div = &divs[0];
 593                partitions[tsgl->nents].length = 0;
 594                tsgl->nents++;
 595        }
 596        partitions[tsgl->nents - 1].length += len_remaining;
 597
 598        /* Set up the sgl entries and fill the data or poison */
 599        sg_init_table(tsgl->sgl, tsgl->nents);
 600        for (i = 0; i < tsgl->nents; i++) {
 601                unsigned int offset = partitions[i].div->offset;
 602                void *addr;
 603
 604                if (partitions[i].div->offset_relative_to_alignmask)
 605                        offset += alignmask;
 606
 607                while (offset + partitions[i].length + TESTMGR_POISON_LEN >
 608                       2 * PAGE_SIZE) {
 609                        if (WARN_ON(offset <= 0))
 610                                return -EINVAL;
 611                        offset /= 2;
 612                }
 613
 614                addr = &tsgl->bufs[i][offset];
 615                sg_set_buf(&tsgl->sgl[i], addr, partitions[i].length);
 616
 617                if (out_divs)
 618                        out_divs[i] = partitions[i].div;
 619
 620                if (data) {
 621                        size_t copy_len, copied;
 622
 623                        copy_len = min(partitions[i].length, data->count);
 624                        copied = copy_from_iter(addr, copy_len, data);
 625                        if (WARN_ON(copied != copy_len))
 626                                return -EINVAL;
 627                        testmgr_poison(addr + copy_len, partitions[i].length +
 628                                       TESTMGR_POISON_LEN - copy_len);
 629                } else {
 630                        testmgr_poison(addr, partitions[i].length +
 631                                       TESTMGR_POISON_LEN);
 632                }
 633        }
 634
 635        sg_mark_end(&tsgl->sgl[tsgl->nents - 1]);
 636        tsgl->sgl_ptr = tsgl->sgl;
 637        memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));
 638        return 0;
 639}
 640
 641/*
 642 * Verify that a scatterlist crypto operation produced the correct output.
 643 *
 644 * @tsgl: scatterlist containing the actual output
 645 * @expected_output: buffer containing the expected output
 646 * @len_to_check: length of @expected_output in bytes
 647 * @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result
 648 * @check_poison: verify that the poison bytes after each chunk are intact?
 649 *
 650 * Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.
 651 */
 652static int verify_correct_output(const struct test_sglist *tsgl,
 653                                 const char *expected_output,
 654                                 unsigned int len_to_check,
 655                                 unsigned int unchecked_prefix_len,
 656                                 bool check_poison)
 657{
 658        unsigned int i;
 659
 660        for (i = 0; i < tsgl->nents; i++) {
 661                struct scatterlist *sg = &tsgl->sgl_ptr[i];
 662                unsigned int len = sg->length;
 663                unsigned int offset = sg->offset;
 664                const char *actual_output;
 665
 666                if (unchecked_prefix_len) {
 667                        if (unchecked_prefix_len >= len) {
 668                                unchecked_prefix_len -= len;
 669                                continue;
 670                        }
 671                        offset += unchecked_prefix_len;
 672                        len -= unchecked_prefix_len;
 673                        unchecked_prefix_len = 0;
 674                }
 675                len = min(len, len_to_check);
 676                actual_output = page_address(sg_page(sg)) + offset;
 677                if (memcmp(expected_output, actual_output, len) != 0)
 678                        return -EINVAL;
 679                if (check_poison &&
 680                    !testmgr_is_poison(actual_output + len, TESTMGR_POISON_LEN))
 681                        return -EOVERFLOW;
 682                len_to_check -= len;
 683                expected_output += len;
 684        }
 685        if (WARN_ON(len_to_check != 0))
 686                return -EINVAL;
 687        return 0;
 688}
 689
 690static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)
 691{
 692        unsigned int i;
 693
 694        for (i = 0; i < tsgl->nents; i++) {
 695                if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)
 696                        return true;
 697                if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)
 698                        return true;
 699                if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)
 700                        return true;
 701        }
 702        return false;
 703}
 704
 705struct cipher_test_sglists {
 706        struct test_sglist src;
 707        struct test_sglist dst;
 708};
 709
 710static struct cipher_test_sglists *alloc_cipher_test_sglists(void)
 711{
 712        struct cipher_test_sglists *tsgls;
 713
 714        tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);
 715        if (!tsgls)
 716                return NULL;
 717
 718        if (init_test_sglist(&tsgls->src) != 0)
 719                goto fail_kfree;
 720        if (init_test_sglist(&tsgls->dst) != 0)
 721                goto fail_destroy_src;
 722
 723        return tsgls;
 724
 725fail_destroy_src:
 726        destroy_test_sglist(&tsgls->src);
 727fail_kfree:
 728        kfree(tsgls);
 729        return NULL;
 730}
 731
 732static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)
 733{
 734        if (tsgls) {
 735                destroy_test_sglist(&tsgls->src);
 736                destroy_test_sglist(&tsgls->dst);
 737                kfree(tsgls);
 738        }
 739}
 740
 741/* Build the src and dst scatterlists for an skcipher or AEAD test */
 742static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,
 743                                     const struct testvec_config *cfg,
 744                                     unsigned int alignmask,
 745                                     unsigned int src_total_len,
 746                                     unsigned int dst_total_len,
 747                                     const struct kvec *inputs,
 748                                     unsigned int nr_inputs)
 749{
 750        struct iov_iter input;
 751        int err;
 752
 753        iov_iter_kvec(&input, WRITE, inputs, nr_inputs, src_total_len);
 754        err = build_test_sglist(&tsgls->src, cfg->src_divs, alignmask,
 755                                cfg->inplace ?
 756                                        max(dst_total_len, src_total_len) :
 757                                        src_total_len,
 758                                &input, NULL);
 759        if (err)
 760                return err;
 761
 762        if (cfg->inplace) {
 763                tsgls->dst.sgl_ptr = tsgls->src.sgl;
 764                tsgls->dst.nents = tsgls->src.nents;
 765                return 0;
 766        }
 767        return build_test_sglist(&tsgls->dst,
 768                                 cfg->dst_divs[0].proportion_of_total ?
 769                                        cfg->dst_divs : cfg->src_divs,
 770                                 alignmask, dst_total_len, NULL, NULL);
 771}
 772
 773/*
 774 * Support for testing passing a misaligned key to setkey():
 775 *
 776 * If cfg->key_offset is set, copy the key into a new buffer at that offset,
 777 * optionally adding alignmask.  Else, just use the key directly.
 778 */
 779static int prepare_keybuf(const u8 *key, unsigned int ksize,
 780                          const struct testvec_config *cfg,
 781                          unsigned int alignmask,
 782                          const u8 **keybuf_ret, const u8 **keyptr_ret)
 783{
 784        unsigned int key_offset = cfg->key_offset;
 785        u8 *keybuf = NULL, *keyptr = (u8 *)key;
 786
 787        if (key_offset != 0) {
 788                if (cfg->key_offset_relative_to_alignmask)
 789                        key_offset += alignmask;
 790                keybuf = kmalloc(key_offset + ksize, GFP_KERNEL);
 791                if (!keybuf)
 792                        return -ENOMEM;
 793                keyptr = keybuf + key_offset;
 794                memcpy(keyptr, key, ksize);
 795        }
 796        *keybuf_ret = keybuf;
 797        *keyptr_ret = keyptr;
 798        return 0;
 799}
 800
 801/* Like setkey_f(tfm, key, ksize), but sometimes misalign the key */
 802#define do_setkey(setkey_f, tfm, key, ksize, cfg, alignmask)            \
 803({                                                                      \
 804        const u8 *keybuf, *keyptr;                                      \
 805        int err;                                                        \
 806                                                                        \
 807        err = prepare_keybuf((key), (ksize), (cfg), (alignmask),        \
 808                             &keybuf, &keyptr);                         \
 809        if (err == 0) {                                                 \
 810                err = setkey_f((tfm), keyptr, (ksize));                 \
 811                kfree(keybuf);                                          \
 812        }                                                               \
 813        err;                                                            \
 814})
 815
 816#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
 817
 818/* Generate a random length in range [0, max_len], but prefer smaller values */
 819static unsigned int generate_random_length(unsigned int max_len)
 820{
 821        unsigned int len = prandom_u32() % (max_len + 1);
 822
 823        switch (prandom_u32() % 4) {
 824        case 0:
 825                return len % 64;
 826        case 1:
 827                return len % 256;
 828        case 2:
 829                return len % 1024;
 830        default:
 831                return len;
 832        }
 833}
 834
 835/* Flip a random bit in the given nonempty data buffer */
 836static void flip_random_bit(u8 *buf, size_t size)
 837{
 838        size_t bitpos;
 839
 840        bitpos = prandom_u32() % (size * 8);
 841        buf[bitpos / 8] ^= 1 << (bitpos % 8);
 842}
 843
 844/* Flip a random byte in the given nonempty data buffer */
 845static void flip_random_byte(u8 *buf, size_t size)
 846{
 847        buf[prandom_u32() % size] ^= 0xff;
 848}
 849
 850/* Sometimes make some random changes to the given nonempty data buffer */
 851static void mutate_buffer(u8 *buf, size_t size)
 852{
 853        size_t num_flips;
 854        size_t i;
 855
 856        /* Sometimes flip some bits */
 857        if (prandom_u32() % 4 == 0) {
 858                num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size * 8);
 859                for (i = 0; i < num_flips; i++)
 860                        flip_random_bit(buf, size);
 861        }
 862
 863        /* Sometimes flip some bytes */
 864        if (prandom_u32() % 4 == 0) {
 865                num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size);
 866                for (i = 0; i < num_flips; i++)
 867                        flip_random_byte(buf, size);
 868        }
 869}
 870
 871/* Randomly generate 'count' bytes, but sometimes make them "interesting" */
 872static void generate_random_bytes(u8 *buf, size_t count)
 873{
 874        u8 b;
 875        u8 increment;
 876        size_t i;
 877
 878        if (count == 0)
 879                return;
 880
 881        switch (prandom_u32() % 8) { /* Choose a generation strategy */
 882        case 0:
 883        case 1:
 884                /* All the same byte, plus optional mutations */
 885                switch (prandom_u32() % 4) {
 886                case 0:
 887                        b = 0x00;
 888                        break;
 889                case 1:
 890                        b = 0xff;
 891                        break;
 892                default:
 893                        b = (u8)prandom_u32();
 894                        break;
 895                }
 896                memset(buf, b, count);
 897                mutate_buffer(buf, count);
 898                break;
 899        case 2:
 900                /* Ascending or descending bytes, plus optional mutations */
 901                increment = (u8)prandom_u32();
 902                b = (u8)prandom_u32();
 903                for (i = 0; i < count; i++, b += increment)
 904                        buf[i] = b;
 905                mutate_buffer(buf, count);
 906                break;
 907        default:
 908                /* Fully random bytes */
 909                for (i = 0; i < count; i++)
 910                        buf[i] = (u8)prandom_u32();
 911        }
 912}
 913
 914static char *generate_random_sgl_divisions(struct test_sg_division *divs,
 915                                           size_t max_divs, char *p, char *end,
 916                                           bool gen_flushes, u32 req_flags)
 917{
 918        struct test_sg_division *div = divs;
 919        unsigned int remaining = TEST_SG_TOTAL;
 920
 921        do {
 922                unsigned int this_len;
 923                const char *flushtype_str;
 924
 925                if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0)
 926                        this_len = remaining;
 927                else
 928                        this_len = 1 + (prandom_u32() % remaining);
 929                div->proportion_of_total = this_len;
 930
 931                if (prandom_u32() % 4 == 0)
 932                        div->offset = (PAGE_SIZE - 128) + (prandom_u32() % 128);
 933                else if (prandom_u32() % 2 == 0)
 934                        div->offset = prandom_u32() % 32;
 935                else
 936                        div->offset = prandom_u32() % PAGE_SIZE;
 937                if (prandom_u32() % 8 == 0)
 938                        div->offset_relative_to_alignmask = true;
 939
 940                div->flush_type = FLUSH_TYPE_NONE;
 941                if (gen_flushes) {
 942                        switch (prandom_u32() % 4) {
 943                        case 0:
 944                                div->flush_type = FLUSH_TYPE_REIMPORT;
 945                                break;
 946                        case 1:
 947                                div->flush_type = FLUSH_TYPE_FLUSH;
 948                                break;
 949                        }
 950                }
 951
 952                if (div->flush_type != FLUSH_TYPE_NONE &&
 953                    !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
 954                    prandom_u32() % 2 == 0)
 955                        div->nosimd = true;
 956
 957                switch (div->flush_type) {
 958                case FLUSH_TYPE_FLUSH:
 959                        if (div->nosimd)
 960                                flushtype_str = "<flush,nosimd>";
 961                        else
 962                                flushtype_str = "<flush>";
 963                        break;
 964                case FLUSH_TYPE_REIMPORT:
 965                        if (div->nosimd)
 966                                flushtype_str = "<reimport,nosimd>";
 967                        else
 968                                flushtype_str = "<reimport>";
 969                        break;
 970                default:
 971                        flushtype_str = "";
 972                        break;
 973                }
 974
 975                BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */
 976                p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str,
 977                               this_len / 100, this_len % 100,
 978                               div->offset_relative_to_alignmask ?
 979                                        "alignmask" : "",
 980                               div->offset, this_len == remaining ? "" : ", ");
 981                remaining -= this_len;
 982                div++;
 983        } while (remaining);
 984
 985        return p;
 986}
 987
 988/* Generate a random testvec_config for fuzz testing */
 989static void generate_random_testvec_config(struct testvec_config *cfg,
 990                                           char *name, size_t max_namelen)
 991{
 992        char *p = name;
 993        char * const end = name + max_namelen;
 994
 995        memset(cfg, 0, sizeof(*cfg));
 996
 997        cfg->name = name;
 998
 999        p += scnprintf(p, end - p, "random:");
1000
1001        if (prandom_u32() % 2 == 0) {
1002                cfg->inplace = true;
1003                p += scnprintf(p, end - p, " inplace");
1004        }
1005
1006        if (prandom_u32() % 2 == 0) {
1007                cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
1008                p += scnprintf(p, end - p, " may_sleep");
1009        }
1010
1011        switch (prandom_u32() % 4) {
1012        case 0:
1013                cfg->finalization_type = FINALIZATION_TYPE_FINAL;
1014                p += scnprintf(p, end - p, " use_final");
1015                break;
1016        case 1:
1017                cfg->finalization_type = FINALIZATION_TYPE_FINUP;
1018                p += scnprintf(p, end - p, " use_finup");
1019                break;
1020        default:
1021                cfg->finalization_type = FINALIZATION_TYPE_DIGEST;
1022                p += scnprintf(p, end - p, " use_digest");
1023                break;
1024        }
1025
1026        if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&
1027            prandom_u32() % 2 == 0) {
1028                cfg->nosimd = true;
1029                p += scnprintf(p, end - p, " nosimd");
1030        }
1031
1032        p += scnprintf(p, end - p, " src_divs=[");
1033        p = generate_random_sgl_divisions(cfg->src_divs,
1034                                          ARRAY_SIZE(cfg->src_divs), p, end,
1035                                          (cfg->finalization_type !=
1036                                           FINALIZATION_TYPE_DIGEST),
1037                                          cfg->req_flags);
1038        p += scnprintf(p, end - p, "]");
1039
1040        if (!cfg->inplace && prandom_u32() % 2 == 0) {
1041                p += scnprintf(p, end - p, " dst_divs=[");
1042                p = generate_random_sgl_divisions(cfg->dst_divs,
1043                                                  ARRAY_SIZE(cfg->dst_divs),
1044                                                  p, end, false,
1045                                                  cfg->req_flags);
1046                p += scnprintf(p, end - p, "]");
1047        }
1048
1049        if (prandom_u32() % 2 == 0) {
1050                cfg->iv_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
1051                p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);
1052        }
1053
1054        if (prandom_u32() % 2 == 0) {
1055                cfg->key_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK);
1056                p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset);
1057        }
1058
1059        WARN_ON_ONCE(!valid_testvec_config(cfg));
1060}
1061
1062static void crypto_disable_simd_for_test(void)
1063{
1064        preempt_disable();
1065        __this_cpu_write(crypto_simd_disabled_for_test, true);
1066}
1067
1068static void crypto_reenable_simd_for_test(void)
1069{
1070        __this_cpu_write(crypto_simd_disabled_for_test, false);
1071        preempt_enable();
1072}
1073
1074/*
1075 * Given an algorithm name, build the name of the generic implementation of that
1076 * algorithm, assuming the usual naming convention.  Specifically, this appends
1077 * "-generic" to every part of the name that is not a template name.  Examples:
1078 *
1079 *      aes => aes-generic
1080 *      cbc(aes) => cbc(aes-generic)
1081 *      cts(cbc(aes)) => cts(cbc(aes-generic))
1082 *      rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)
1083 *
1084 * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long
1085 */
1086static int build_generic_driver_name(const char *algname,
1087                                     char driver_name[CRYPTO_MAX_ALG_NAME])
1088{
1089        const char *in = algname;
1090        char *out = driver_name;
1091        size_t len = strlen(algname);
1092
1093        if (len >= CRYPTO_MAX_ALG_NAME)
1094                goto too_long;
1095        do {
1096                const char *in_saved = in;
1097
1098                while (*in && *in != '(' && *in != ')' && *in != ',')
1099                        *out++ = *in++;
1100                if (*in != '(' && in > in_saved) {
1101                        len += 8;
1102                        if (len >= CRYPTO_MAX_ALG_NAME)
1103                                goto too_long;
1104                        memcpy(out, "-generic", 8);
1105                        out += 8;
1106                }
1107        } while ((*out++ = *in++) != '\0');
1108        return 0;
1109
1110too_long:
1111        pr_err("alg: generic driver name for \"%s\" would be too long\n",
1112               algname);
1113        return -ENAMETOOLONG;
1114}
1115#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1116static void crypto_disable_simd_for_test(void)
1117{
1118}
1119
1120static void crypto_reenable_simd_for_test(void)
1121{
1122}
1123#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1124
1125static int build_hash_sglist(struct test_sglist *tsgl,
1126                             const struct hash_testvec *vec,
1127                             const struct testvec_config *cfg,
1128                             unsigned int alignmask,
1129                             const struct test_sg_division *divs[XBUFSIZE])
1130{
1131        struct kvec kv;
1132        struct iov_iter input;
1133
1134        kv.iov_base = (void *)vec->plaintext;
1135        kv.iov_len = vec->psize;
1136        iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize);
1137        return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
1138                                 &input, divs);
1139}
1140
1141static int check_hash_result(const char *type,
1142                             const u8 *result, unsigned int digestsize,
1143                             const struct hash_testvec *vec,
1144                             const char *vec_name,
1145                             const char *driver,
1146                             const struct testvec_config *cfg)
1147{
1148        if (memcmp(result, vec->digest, digestsize) != 0) {
1149                pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
1150                       type, driver, vec_name, cfg->name);
1151                return -EINVAL;
1152        }
1153        if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
1154                pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
1155                       type, driver, vec_name, cfg->name);
1156                return -EOVERFLOW;
1157        }
1158        return 0;
1159}
1160
1161static inline int check_shash_op(const char *op, int err,
1162                                 const char *driver, const char *vec_name,
1163                                 const struct testvec_config *cfg)
1164{
1165        if (err)
1166                pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1167                       driver, op, err, vec_name, cfg->name);
1168        return err;
1169}
1170
1171/* Test one hash test vector in one configuration, using the shash API */
1172static int test_shash_vec_cfg(const struct hash_testvec *vec,
1173                              const char *vec_name,
1174                              const struct testvec_config *cfg,
1175                              struct shash_desc *desc,
1176                              struct test_sglist *tsgl,
1177                              u8 *hashstate)
1178{
1179        struct crypto_shash *tfm = desc->tfm;
1180        const unsigned int alignmask = crypto_shash_alignmask(tfm);
1181        const unsigned int digestsize = crypto_shash_digestsize(tfm);
1182        const unsigned int statesize = crypto_shash_statesize(tfm);
1183        const char *driver = crypto_shash_driver_name(tfm);
1184        const struct test_sg_division *divs[XBUFSIZE];
1185        unsigned int i;
1186        u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1187        int err;
1188
1189        /* Set the key, if specified */
1190        if (vec->ksize) {
1191                err = do_setkey(crypto_shash_setkey, tfm, vec->key, vec->ksize,
1192                                cfg, alignmask);
1193                if (err) {
1194                        if (err == vec->setkey_error)
1195                                return 0;
1196                        pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1197                               driver, vec_name, vec->setkey_error, err,
1198                               crypto_shash_get_flags(tfm));
1199                        return err;
1200                }
1201                if (vec->setkey_error) {
1202                        pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1203                               driver, vec_name, vec->setkey_error);
1204                        return -EINVAL;
1205                }
1206        }
1207
1208        /* Build the scatterlist for the source data */
1209        err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
1210        if (err) {
1211                pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1212                       driver, vec_name, cfg->name);
1213                return err;
1214        }
1215
1216        /* Do the actual hashing */
1217
1218        testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1219        testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1220
1221        if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1222            vec->digest_error) {
1223                /* Just using digest() */
1224                if (tsgl->nents != 1)
1225                        return 0;
1226                if (cfg->nosimd)
1227                        crypto_disable_simd_for_test();
1228                err = crypto_shash_digest(desc, sg_virt(&tsgl->sgl[0]),
1229                                          tsgl->sgl[0].length, result);
1230                if (cfg->nosimd)
1231                        crypto_reenable_simd_for_test();
1232                if (err) {
1233                        if (err == vec->digest_error)
1234                                return 0;
1235                        pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1236                               driver, vec_name, vec->digest_error, err,
1237                               cfg->name);
1238                        return err;
1239                }
1240                if (vec->digest_error) {
1241                        pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1242                               driver, vec_name, vec->digest_error, cfg->name);
1243                        return -EINVAL;
1244                }
1245                goto result_ready;
1246        }
1247
1248        /* Using init(), zero or more update(), then final() or finup() */
1249
1250        if (cfg->nosimd)
1251                crypto_disable_simd_for_test();
1252        err = crypto_shash_init(desc);
1253        if (cfg->nosimd)
1254                crypto_reenable_simd_for_test();
1255        err = check_shash_op("init", err, driver, vec_name, cfg);
1256        if (err)
1257                return err;
1258
1259        for (i = 0; i < tsgl->nents; i++) {
1260                if (i + 1 == tsgl->nents &&
1261                    cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
1262                        if (divs[i]->nosimd)
1263                                crypto_disable_simd_for_test();
1264                        err = crypto_shash_finup(desc, sg_virt(&tsgl->sgl[i]),
1265                                                 tsgl->sgl[i].length, result);
1266                        if (divs[i]->nosimd)
1267                                crypto_reenable_simd_for_test();
1268                        err = check_shash_op("finup", err, driver, vec_name,
1269                                             cfg);
1270                        if (err)
1271                                return err;
1272                        goto result_ready;
1273                }
1274                if (divs[i]->nosimd)
1275                        crypto_disable_simd_for_test();
1276                err = crypto_shash_update(desc, sg_virt(&tsgl->sgl[i]),
1277                                          tsgl->sgl[i].length);
1278                if (divs[i]->nosimd)
1279                        crypto_reenable_simd_for_test();
1280                err = check_shash_op("update", err, driver, vec_name, cfg);
1281                if (err)
1282                        return err;
1283                if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1284                        /* Test ->export() and ->import() */
1285                        testmgr_poison(hashstate + statesize,
1286                                       TESTMGR_POISON_LEN);
1287                        err = crypto_shash_export(desc, hashstate);
1288                        err = check_shash_op("export", err, driver, vec_name,
1289                                             cfg);
1290                        if (err)
1291                                return err;
1292                        if (!testmgr_is_poison(hashstate + statesize,
1293                                               TESTMGR_POISON_LEN)) {
1294                                pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1295                                       driver, vec_name, cfg->name);
1296                                return -EOVERFLOW;
1297                        }
1298                        testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
1299                        err = crypto_shash_import(desc, hashstate);
1300                        err = check_shash_op("import", err, driver, vec_name,
1301                                             cfg);
1302                        if (err)
1303                                return err;
1304                }
1305        }
1306
1307        if (cfg->nosimd)
1308                crypto_disable_simd_for_test();
1309        err = crypto_shash_final(desc, result);
1310        if (cfg->nosimd)
1311                crypto_reenable_simd_for_test();
1312        err = check_shash_op("final", err, driver, vec_name, cfg);
1313        if (err)
1314                return err;
1315result_ready:
1316        return check_hash_result("shash", result, digestsize, vec, vec_name,
1317                                 driver, cfg);
1318}
1319
1320static int do_ahash_op(int (*op)(struct ahash_request *req),
1321                       struct ahash_request *req,
1322                       struct crypto_wait *wait, bool nosimd)
1323{
1324        int err;
1325
1326        if (nosimd)
1327                crypto_disable_simd_for_test();
1328
1329        err = op(req);
1330
1331        if (nosimd)
1332                crypto_reenable_simd_for_test();
1333
1334        return crypto_wait_req(err, wait);
1335}
1336
1337static int check_nonfinal_ahash_op(const char *op, int err,
1338                                   u8 *result, unsigned int digestsize,
1339                                   const char *driver, const char *vec_name,
1340                                   const struct testvec_config *cfg)
1341{
1342        if (err) {
1343                pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
1344                       driver, op, err, vec_name, cfg->name);
1345                return err;
1346        }
1347        if (!testmgr_is_poison(result, digestsize)) {
1348                pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
1349                       driver, op, vec_name, cfg->name);
1350                return -EINVAL;
1351        }
1352        return 0;
1353}
1354
1355/* Test one hash test vector in one configuration, using the ahash API */
1356static int test_ahash_vec_cfg(const struct hash_testvec *vec,
1357                              const char *vec_name,
1358                              const struct testvec_config *cfg,
1359                              struct ahash_request *req,
1360                              struct test_sglist *tsgl,
1361                              u8 *hashstate)
1362{
1363        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1364        const unsigned int alignmask = crypto_ahash_alignmask(tfm);
1365        const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1366        const unsigned int statesize = crypto_ahash_statesize(tfm);
1367        const char *driver = crypto_ahash_driver_name(tfm);
1368        const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1369        const struct test_sg_division *divs[XBUFSIZE];
1370        DECLARE_CRYPTO_WAIT(wait);
1371        unsigned int i;
1372        struct scatterlist *pending_sgl;
1373        unsigned int pending_len;
1374        u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
1375        int err;
1376
1377        /* Set the key, if specified */
1378        if (vec->ksize) {
1379                err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize,
1380                                cfg, alignmask);
1381                if (err) {
1382                        if (err == vec->setkey_error)
1383                                return 0;
1384                        pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1385                               driver, vec_name, vec->setkey_error, err,
1386                               crypto_ahash_get_flags(tfm));
1387                        return err;
1388                }
1389                if (vec->setkey_error) {
1390                        pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1391                               driver, vec_name, vec->setkey_error);
1392                        return -EINVAL;
1393                }
1394        }
1395
1396        /* Build the scatterlist for the source data */
1397        err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
1398        if (err) {
1399                pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
1400                       driver, vec_name, cfg->name);
1401                return err;
1402        }
1403
1404        /* Do the actual hashing */
1405
1406        testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1407        testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
1408
1409        if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
1410            vec->digest_error) {
1411                /* Just using digest() */
1412                ahash_request_set_callback(req, req_flags, crypto_req_done,
1413                                           &wait);
1414                ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize);
1415                err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd);
1416                if (err) {
1417                        if (err == vec->digest_error)
1418                                return 0;
1419                        pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
1420                               driver, vec_name, vec->digest_error, err,
1421                               cfg->name);
1422                        return err;
1423                }
1424                if (vec->digest_error) {
1425                        pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
1426                               driver, vec_name, vec->digest_error, cfg->name);
1427                        return -EINVAL;
1428                }
1429                goto result_ready;
1430        }
1431
1432        /* Using init(), zero or more update(), then final() or finup() */
1433
1434        ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1435        ahash_request_set_crypt(req, NULL, result, 0);
1436        err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
1437        err = check_nonfinal_ahash_op("init", err, result, digestsize,
1438                                      driver, vec_name, cfg);
1439        if (err)
1440                return err;
1441
1442        pending_sgl = NULL;
1443        pending_len = 0;
1444        for (i = 0; i < tsgl->nents; i++) {
1445                if (divs[i]->flush_type != FLUSH_TYPE_NONE &&
1446                    pending_sgl != NULL) {
1447                        /* update() with the pending data */
1448                        ahash_request_set_callback(req, req_flags,
1449                                                   crypto_req_done, &wait);
1450                        ahash_request_set_crypt(req, pending_sgl, result,
1451                                                pending_len);
1452                        err = do_ahash_op(crypto_ahash_update, req, &wait,
1453                                          divs[i]->nosimd);
1454                        err = check_nonfinal_ahash_op("update", err,
1455                                                      result, digestsize,
1456                                                      driver, vec_name, cfg);
1457                        if (err)
1458                                return err;
1459                        pending_sgl = NULL;
1460                        pending_len = 0;
1461                }
1462                if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
1463                        /* Test ->export() and ->import() */
1464                        testmgr_poison(hashstate + statesize,
1465                                       TESTMGR_POISON_LEN);
1466                        err = crypto_ahash_export(req, hashstate);
1467                        err = check_nonfinal_ahash_op("export", err,
1468                                                      result, digestsize,
1469                                                      driver, vec_name, cfg);
1470                        if (err)
1471                                return err;
1472                        if (!testmgr_is_poison(hashstate + statesize,
1473                                               TESTMGR_POISON_LEN)) {
1474                                pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
1475                                       driver, vec_name, cfg->name);
1476                                return -EOVERFLOW;
1477                        }
1478
1479                        testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
1480                        err = crypto_ahash_import(req, hashstate);
1481                        err = check_nonfinal_ahash_op("import", err,
1482                                                      result, digestsize,
1483                                                      driver, vec_name, cfg);
1484                        if (err)
1485                                return err;
1486                }
1487                if (pending_sgl == NULL)
1488                        pending_sgl = &tsgl->sgl[i];
1489                pending_len += tsgl->sgl[i].length;
1490        }
1491
1492        ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
1493        ahash_request_set_crypt(req, pending_sgl, result, pending_len);
1494        if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
1495                /* finish with update() and final() */
1496                err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
1497                err = check_nonfinal_ahash_op("update", err, result, digestsize,
1498                                              driver, vec_name, cfg);
1499                if (err)
1500                        return err;
1501                err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
1502                if (err) {
1503                        pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
1504                               driver, err, vec_name, cfg->name);
1505                        return err;
1506                }
1507        } else {
1508                /* finish with finup() */
1509                err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
1510                if (err) {
1511                        pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
1512                               driver, err, vec_name, cfg->name);
1513                        return err;
1514                }
1515        }
1516
1517result_ready:
1518        return check_hash_result("ahash", result, digestsize, vec, vec_name,
1519                                 driver, cfg);
1520}
1521
1522static int test_hash_vec_cfg(const struct hash_testvec *vec,
1523                             const char *vec_name,
1524                             const struct testvec_config *cfg,
1525                             struct ahash_request *req,
1526                             struct shash_desc *desc,
1527                             struct test_sglist *tsgl,
1528                             u8 *hashstate)
1529{
1530        int err;
1531
1532        /*
1533         * For algorithms implemented as "shash", most bugs will be detected by
1534         * both the shash and ahash tests.  Test the shash API first so that the
1535         * failures involve less indirection, so are easier to debug.
1536         */
1537
1538        if (desc) {
1539                err = test_shash_vec_cfg(vec, vec_name, cfg, desc, tsgl,
1540                                         hashstate);
1541                if (err)
1542                        return err;
1543        }
1544
1545        return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate);
1546}
1547
1548static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num,
1549                         struct ahash_request *req, struct shash_desc *desc,
1550                         struct test_sglist *tsgl, u8 *hashstate)
1551{
1552        char vec_name[16];
1553        unsigned int i;
1554        int err;
1555
1556        sprintf(vec_name, "%u", vec_num);
1557
1558        for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
1559                err = test_hash_vec_cfg(vec, vec_name,
1560                                        &default_hash_testvec_configs[i],
1561                                        req, desc, tsgl, hashstate);
1562                if (err)
1563                        return err;
1564        }
1565
1566#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1567        if (!noextratests) {
1568                struct testvec_config cfg;
1569                char cfgname[TESTVEC_CONFIG_NAMELEN];
1570
1571                for (i = 0; i < fuzz_iterations; i++) {
1572                        generate_random_testvec_config(&cfg, cfgname,
1573                                                       sizeof(cfgname));
1574                        err = test_hash_vec_cfg(vec, vec_name, &cfg,
1575                                                req, desc, tsgl, hashstate);
1576                        if (err)
1577                                return err;
1578                        cond_resched();
1579                }
1580        }
1581#endif
1582        return 0;
1583}
1584
1585#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
1586/*
1587 * Generate a hash test vector from the given implementation.
1588 * Assumes the buffers in 'vec' were already allocated.
1589 */
1590static void generate_random_hash_testvec(struct shash_desc *desc,
1591                                         struct hash_testvec *vec,
1592                                         unsigned int maxkeysize,
1593                                         unsigned int maxdatasize,
1594                                         char *name, size_t max_namelen)
1595{
1596        /* Data */
1597        vec->psize = generate_random_length(maxdatasize);
1598        generate_random_bytes((u8 *)vec->plaintext, vec->psize);
1599
1600        /*
1601         * Key: length in range [1, maxkeysize], but usually choose maxkeysize.
1602         * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.
1603         */
1604        vec->setkey_error = 0;
1605        vec->ksize = 0;
1606        if (maxkeysize) {
1607                vec->ksize = maxkeysize;
1608                if (prandom_u32() % 4 == 0)
1609                        vec->ksize = 1 + (prandom_u32() % maxkeysize);
1610                generate_random_bytes((u8 *)vec->key, vec->ksize);
1611
1612                vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,
1613                                                        vec->ksize);
1614                /* If the key couldn't be set, no need to continue to digest. */
1615                if (vec->setkey_error)
1616                        goto done;
1617        }
1618
1619        /* Digest */
1620        vec->digest_error = crypto_shash_digest(desc, vec->plaintext,
1621                                                vec->psize, (u8 *)vec->digest);
1622done:
1623        snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"",
1624                 vec->psize, vec->ksize);
1625}
1626
1627/*
1628 * Test the hash algorithm represented by @req against the corresponding generic
1629 * implementation, if one is available.
1630 */
1631static int test_hash_vs_generic_impl(const char *generic_driver,
1632                                     unsigned int maxkeysize,
1633                                     struct ahash_request *req,
1634                                     struct shash_desc *desc,
1635                                     struct test_sglist *tsgl,
1636                                     u8 *hashstate)
1637{
1638        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1639        const unsigned int digestsize = crypto_ahash_digestsize(tfm);
1640        const unsigned int blocksize = crypto_ahash_blocksize(tfm);
1641        const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
1642        const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
1643        const char *driver = crypto_ahash_driver_name(tfm);
1644        char _generic_driver[CRYPTO_MAX_ALG_NAME];
1645        struct crypto_shash *generic_tfm = NULL;
1646        struct shash_desc *generic_desc = NULL;
1647        unsigned int i;
1648        struct hash_testvec vec = { 0 };
1649        char vec_name[64];
1650        struct testvec_config *cfg;
1651        char cfgname[TESTVEC_CONFIG_NAMELEN];
1652        int err;
1653
1654        if (noextratests)
1655                return 0;
1656
1657        if (!generic_driver) { /* Use default naming convention? */
1658                err = build_generic_driver_name(algname, _generic_driver);
1659                if (err)
1660                        return err;
1661                generic_driver = _generic_driver;
1662        }
1663
1664        if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
1665                return 0;
1666
1667        generic_tfm = crypto_alloc_shash(generic_driver, 0, 0);
1668        if (IS_ERR(generic_tfm)) {
1669                err = PTR_ERR(generic_tfm);
1670                if (err == -ENOENT) {
1671                        pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",
1672                                driver, generic_driver);
1673                        return 0;
1674                }
1675                pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",
1676                       generic_driver, algname, err);
1677                return err;
1678        }
1679
1680        cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1681        if (!cfg) {
1682                err = -ENOMEM;
1683                goto out;
1684        }
1685
1686        generic_desc = kzalloc(sizeof(*desc) +
1687                               crypto_shash_descsize(generic_tfm), GFP_KERNEL);
1688        if (!generic_desc) {
1689                err = -ENOMEM;
1690                goto out;
1691        }
1692        generic_desc->tfm = generic_tfm;
1693
1694        /* Check the algorithm properties for consistency. */
1695
1696        if (digestsize != crypto_shash_digestsize(generic_tfm)) {
1697                pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",
1698                       driver, digestsize,
1699                       crypto_shash_digestsize(generic_tfm));
1700                err = -EINVAL;
1701                goto out;
1702        }
1703
1704        if (blocksize != crypto_shash_blocksize(generic_tfm)) {
1705                pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",
1706                       driver, blocksize, crypto_shash_blocksize(generic_tfm));
1707                err = -EINVAL;
1708                goto out;
1709        }
1710
1711        /*
1712         * Now generate test vectors using the generic implementation, and test
1713         * the other implementation against them.
1714         */
1715
1716        vec.key = kmalloc(maxkeysize, GFP_KERNEL);
1717        vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);
1718        vec.digest = kmalloc(digestsize, GFP_KERNEL);
1719        if (!vec.key || !vec.plaintext || !vec.digest) {
1720                err = -ENOMEM;
1721                goto out;
1722        }
1723
1724        for (i = 0; i < fuzz_iterations * 8; i++) {
1725                generate_random_hash_testvec(generic_desc, &vec,
1726                                             maxkeysize, maxdatasize,
1727                                             vec_name, sizeof(vec_name));
1728                generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
1729
1730                err = test_hash_vec_cfg(&vec, vec_name, cfg,
1731                                        req, desc, tsgl, hashstate);
1732                if (err)
1733                        goto out;
1734                cond_resched();
1735        }
1736        err = 0;
1737out:
1738        kfree(cfg);
1739        kfree(vec.key);
1740        kfree(vec.plaintext);
1741        kfree(vec.digest);
1742        crypto_free_shash(generic_tfm);
1743        kfree_sensitive(generic_desc);
1744        return err;
1745}
1746#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1747static int test_hash_vs_generic_impl(const char *generic_driver,
1748                                     unsigned int maxkeysize,
1749                                     struct ahash_request *req,
1750                                     struct shash_desc *desc,
1751                                     struct test_sglist *tsgl,
1752                                     u8 *hashstate)
1753{
1754        return 0;
1755}
1756#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
1757
1758static int alloc_shash(const char *driver, u32 type, u32 mask,
1759                       struct crypto_shash **tfm_ret,
1760                       struct shash_desc **desc_ret)
1761{
1762        struct crypto_shash *tfm;
1763        struct shash_desc *desc;
1764
1765        tfm = crypto_alloc_shash(driver, type, mask);
1766        if (IS_ERR(tfm)) {
1767                if (PTR_ERR(tfm) == -ENOENT) {
1768                        /*
1769                         * This algorithm is only available through the ahash
1770                         * API, not the shash API, so skip the shash tests.
1771                         */
1772                        return 0;
1773                }
1774                pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
1775                       driver, PTR_ERR(tfm));
1776                return PTR_ERR(tfm);
1777        }
1778
1779        desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
1780        if (!desc) {
1781                crypto_free_shash(tfm);
1782                return -ENOMEM;
1783        }
1784        desc->tfm = tfm;
1785
1786        *tfm_ret = tfm;
1787        *desc_ret = desc;
1788        return 0;
1789}
1790
1791static int __alg_test_hash(const struct hash_testvec *vecs,
1792                           unsigned int num_vecs, const char *driver,
1793                           u32 type, u32 mask,
1794                           const char *generic_driver, unsigned int maxkeysize)
1795{
1796        struct crypto_ahash *atfm = NULL;
1797        struct ahash_request *req = NULL;
1798        struct crypto_shash *stfm = NULL;
1799        struct shash_desc *desc = NULL;
1800        struct test_sglist *tsgl = NULL;
1801        u8 *hashstate = NULL;
1802        unsigned int statesize;
1803        unsigned int i;
1804        int err;
1805
1806        /*
1807         * Always test the ahash API.  This works regardless of whether the
1808         * algorithm is implemented as ahash or shash.
1809         */
1810
1811        atfm = crypto_alloc_ahash(driver, type, mask);
1812        if (IS_ERR(atfm)) {
1813                pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
1814                       driver, PTR_ERR(atfm));
1815                return PTR_ERR(atfm);
1816        }
1817        driver = crypto_ahash_driver_name(atfm);
1818
1819        req = ahash_request_alloc(atfm, GFP_KERNEL);
1820        if (!req) {
1821                pr_err("alg: hash: failed to allocate request for %s\n",
1822                       driver);
1823                err = -ENOMEM;
1824                goto out;
1825        }
1826
1827        /*
1828         * If available also test the shash API, to cover corner cases that may
1829         * be missed by testing the ahash API only.
1830         */
1831        err = alloc_shash(driver, type, mask, &stfm, &desc);
1832        if (err)
1833                goto out;
1834
1835        tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
1836        if (!tsgl || init_test_sglist(tsgl) != 0) {
1837                pr_err("alg: hash: failed to allocate test buffers for %s\n",
1838                       driver);
1839                kfree(tsgl);
1840                tsgl = NULL;
1841                err = -ENOMEM;
1842                goto out;
1843        }
1844
1845        statesize = crypto_ahash_statesize(atfm);
1846        if (stfm)
1847                statesize = max(statesize, crypto_shash_statesize(stfm));
1848        hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
1849        if (!hashstate) {
1850                pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
1851                       driver);
1852                err = -ENOMEM;
1853                goto out;
1854        }
1855
1856        for (i = 0; i < num_vecs; i++) {
1857                err = test_hash_vec(&vecs[i], i, req, desc, tsgl, hashstate);
1858                if (err)
1859                        goto out;
1860                cond_resched();
1861        }
1862        err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req,
1863                                        desc, tsgl, hashstate);
1864out:
1865        kfree(hashstate);
1866        if (tsgl) {
1867                destroy_test_sglist(tsgl);
1868                kfree(tsgl);
1869        }
1870        kfree(desc);
1871        crypto_free_shash(stfm);
1872        ahash_request_free(req);
1873        crypto_free_ahash(atfm);
1874        return err;
1875}
1876
1877static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
1878                         u32 type, u32 mask)
1879{
1880        const struct hash_testvec *template = desc->suite.hash.vecs;
1881        unsigned int tcount = desc->suite.hash.count;
1882        unsigned int nr_unkeyed, nr_keyed;
1883        unsigned int maxkeysize = 0;
1884        int err;
1885
1886        /*
1887         * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests
1888         * first, before setting a key on the tfm.  To make this easier, we
1889         * require that the unkeyed test vectors (if any) are listed first.
1890         */
1891
1892        for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {
1893                if (template[nr_unkeyed].ksize)
1894                        break;
1895        }
1896        for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {
1897                if (!template[nr_unkeyed + nr_keyed].ksize) {
1898                        pr_err("alg: hash: test vectors for %s out of order, "
1899                               "unkeyed ones must come first\n", desc->alg);
1900                        return -EINVAL;
1901                }
1902                maxkeysize = max_t(unsigned int, maxkeysize,
1903                                   template[nr_unkeyed + nr_keyed].ksize);
1904        }
1905
1906        err = 0;
1907        if (nr_unkeyed) {
1908                err = __alg_test_hash(template, nr_unkeyed, driver, type, mask,
1909                                      desc->generic_driver, maxkeysize);
1910                template += nr_unkeyed;
1911        }
1912
1913        if (!err && nr_keyed)
1914                err = __alg_test_hash(template, nr_keyed, driver, type, mask,
1915                                      desc->generic_driver, maxkeysize);
1916
1917        return err;
1918}
1919
1920static int test_aead_vec_cfg(int enc, const struct aead_testvec *vec,
1921                             const char *vec_name,
1922                             const struct testvec_config *cfg,
1923                             struct aead_request *req,
1924                             struct cipher_test_sglists *tsgls)
1925{
1926        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1927        const unsigned int alignmask = crypto_aead_alignmask(tfm);
1928        const unsigned int ivsize = crypto_aead_ivsize(tfm);
1929        const unsigned int authsize = vec->clen - vec->plen;
1930        const char *driver = crypto_aead_driver_name(tfm);
1931        const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
1932        const char *op = enc ? "encryption" : "decryption";
1933        DECLARE_CRYPTO_WAIT(wait);
1934        u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
1935        u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
1936                 cfg->iv_offset +
1937                 (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
1938        struct kvec input[2];
1939        int err;
1940
1941        /* Set the key */
1942        if (vec->wk)
1943                crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1944        else
1945                crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
1946
1947        err = do_setkey(crypto_aead_setkey, tfm, vec->key, vec->klen,
1948                        cfg, alignmask);
1949        if (err && err != vec->setkey_error) {
1950                pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
1951                       driver, vec_name, vec->setkey_error, err,
1952                       crypto_aead_get_flags(tfm));
1953                return err;
1954        }
1955        if (!err && vec->setkey_error) {
1956                pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
1957                       driver, vec_name, vec->setkey_error);
1958                return -EINVAL;
1959        }
1960
1961        /* Set the authentication tag size */
1962        err = crypto_aead_setauthsize(tfm, authsize);
1963        if (err && err != vec->setauthsize_error) {
1964                pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",
1965                       driver, vec_name, vec->setauthsize_error, err);
1966                return err;
1967        }
1968        if (!err && vec->setauthsize_error) {
1969                pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",
1970                       driver, vec_name, vec->setauthsize_error);
1971                return -EINVAL;
1972        }
1973
1974        if (vec->setkey_error || vec->setauthsize_error)
1975                return 0;
1976
1977        /* The IV must be copied to a buffer, as the algorithm may modify it */
1978        if (WARN_ON(ivsize > MAX_IVLEN))
1979                return -EINVAL;
1980        if (vec->iv)
1981                memcpy(iv, vec->iv, ivsize);
1982        else
1983                memset(iv, 0, ivsize);
1984
1985        /* Build the src/dst scatterlists */
1986        input[0].iov_base = (void *)vec->assoc;
1987        input[0].iov_len = vec->alen;
1988        input[1].iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
1989        input[1].iov_len = enc ? vec->plen : vec->clen;
1990        err = build_cipher_test_sglists(tsgls, cfg, alignmask,
1991                                        vec->alen + (enc ? vec->plen :
1992                                                     vec->clen),
1993                                        vec->alen + (enc ? vec->clen :
1994                                                     vec->plen),
1995                                        input, 2);
1996        if (err) {
1997                pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
1998                       driver, op, vec_name, cfg->name);
1999                return err;
2000        }
2001
2002        /* Do the actual encryption or decryption */
2003        testmgr_poison(req->__ctx, crypto_aead_reqsize(tfm));
2004        aead_request_set_callback(req, req_flags, crypto_req_done, &wait);
2005        aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2006                               enc ? vec->plen : vec->clen, iv);
2007        aead_request_set_ad(req, vec->alen);
2008        if (cfg->nosimd)
2009                crypto_disable_simd_for_test();
2010        err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
2011        if (cfg->nosimd)
2012                crypto_reenable_simd_for_test();
2013        err = crypto_wait_req(err, &wait);
2014
2015        /* Check that the algorithm didn't overwrite things it shouldn't have */
2016        if (req->cryptlen != (enc ? vec->plen : vec->clen) ||
2017            req->assoclen != vec->alen ||
2018            req->iv != iv ||
2019            req->src != tsgls->src.sgl_ptr ||
2020            req->dst != tsgls->dst.sgl_ptr ||
2021            crypto_aead_reqtfm(req) != tfm ||
2022            req->base.complete != crypto_req_done ||
2023            req->base.flags != req_flags ||
2024            req->base.data != &wait) {
2025                pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2026                       driver, op, vec_name, cfg->name);
2027                if (req->cryptlen != (enc ? vec->plen : vec->clen))
2028                        pr_err("alg: aead: changed 'req->cryptlen'\n");
2029                if (req->assoclen != vec->alen)
2030                        pr_err("alg: aead: changed 'req->assoclen'\n");
2031                if (req->iv != iv)
2032                        pr_err("alg: aead: changed 'req->iv'\n");
2033                if (req->src != tsgls->src.sgl_ptr)
2034                        pr_err("alg: aead: changed 'req->src'\n");
2035                if (req->dst != tsgls->dst.sgl_ptr)
2036                        pr_err("alg: aead: changed 'req->dst'\n");
2037                if (crypto_aead_reqtfm(req) != tfm)
2038                        pr_err("alg: aead: changed 'req->base.tfm'\n");
2039                if (req->base.complete != crypto_req_done)
2040                        pr_err("alg: aead: changed 'req->base.complete'\n");
2041                if (req->base.flags != req_flags)
2042                        pr_err("alg: aead: changed 'req->base.flags'\n");
2043                if (req->base.data != &wait)
2044                        pr_err("alg: aead: changed 'req->base.data'\n");
2045                return -EINVAL;
2046        }
2047        if (is_test_sglist_corrupted(&tsgls->src)) {
2048                pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2049                       driver, op, vec_name, cfg->name);
2050                return -EINVAL;
2051        }
2052        if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2053            is_test_sglist_corrupted(&tsgls->dst)) {
2054                pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2055                       driver, op, vec_name, cfg->name);
2056                return -EINVAL;
2057        }
2058
2059        /* Check for unexpected success or failure, or wrong error code */
2060        if ((err == 0 && vec->novrfy) ||
2061            (err != vec->crypt_error && !(err == -EBADMSG && vec->novrfy))) {
2062                char expected_error[32];
2063
2064                if (vec->novrfy &&
2065                    vec->crypt_error != 0 && vec->crypt_error != -EBADMSG)
2066                        sprintf(expected_error, "-EBADMSG or %d",
2067                                vec->crypt_error);
2068                else if (vec->novrfy)
2069                        sprintf(expected_error, "-EBADMSG");
2070                else
2071                        sprintf(expected_error, "%d", vec->crypt_error);
2072                if (err) {
2073                        pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%s, actual_error=%d, cfg=\"%s\"\n",
2074                               driver, op, vec_name, expected_error, err,
2075                               cfg->name);
2076                        return err;
2077                }
2078                pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%s, cfg=\"%s\"\n",
2079                       driver, op, vec_name, expected_error, cfg->name);
2080                return -EINVAL;
2081        }
2082        if (err) /* Expectedly failed. */
2083                return 0;
2084
2085        /* Check for the correct output (ciphertext or plaintext) */
2086        err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2087                                    enc ? vec->clen : vec->plen,
2088                                    vec->alen, enc || !cfg->inplace);
2089        if (err == -EOVERFLOW) {
2090                pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2091                       driver, op, vec_name, cfg->name);
2092                return err;
2093        }
2094        if (err) {
2095                pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2096                       driver, op, vec_name, cfg->name);
2097                return err;
2098        }
2099
2100        return 0;
2101}
2102
2103static int test_aead_vec(int enc, const struct aead_testvec *vec,
2104                         unsigned int vec_num, struct aead_request *req,
2105                         struct cipher_test_sglists *tsgls)
2106{
2107        char vec_name[16];
2108        unsigned int i;
2109        int err;
2110
2111        if (enc && vec->novrfy)
2112                return 0;
2113
2114        sprintf(vec_name, "%u", vec_num);
2115
2116        for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2117                err = test_aead_vec_cfg(enc, vec, vec_name,
2118                                        &default_cipher_testvec_configs[i],
2119                                        req, tsgls);
2120                if (err)
2121                        return err;
2122        }
2123
2124#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2125        if (!noextratests) {
2126                struct testvec_config cfg;
2127                char cfgname[TESTVEC_CONFIG_NAMELEN];
2128
2129                for (i = 0; i < fuzz_iterations; i++) {
2130                        generate_random_testvec_config(&cfg, cfgname,
2131                                                       sizeof(cfgname));
2132                        err = test_aead_vec_cfg(enc, vec, vec_name,
2133                                                &cfg, req, tsgls);
2134                        if (err)
2135                                return err;
2136                        cond_resched();
2137                }
2138        }
2139#endif
2140        return 0;
2141}
2142
2143#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2144
2145struct aead_extra_tests_ctx {
2146        struct aead_request *req;
2147        struct crypto_aead *tfm;
2148        const struct alg_test_desc *test_desc;
2149        struct cipher_test_sglists *tsgls;
2150        unsigned int maxdatasize;
2151        unsigned int maxkeysize;
2152
2153        struct aead_testvec vec;
2154        char vec_name[64];
2155        char cfgname[TESTVEC_CONFIG_NAMELEN];
2156        struct testvec_config cfg;
2157};
2158
2159/*
2160 * Make at least one random change to a (ciphertext, AAD) pair.  "Ciphertext"
2161 * here means the full ciphertext including the authentication tag.  The
2162 * authentication tag (and hence also the ciphertext) is assumed to be nonempty.
2163 */
2164static void mutate_aead_message(struct aead_testvec *vec, bool aad_iv,
2165                                unsigned int ivsize)
2166{
2167        const unsigned int aad_tail_size = aad_iv ? ivsize : 0;
2168        const unsigned int authsize = vec->clen - vec->plen;
2169
2170        if (prandom_u32() % 2 == 0 && vec->alen > aad_tail_size) {
2171                 /* Mutate the AAD */
2172                flip_random_bit((u8 *)vec->assoc, vec->alen - aad_tail_size);
2173                if (prandom_u32() % 2 == 0)
2174                        return;
2175        }
2176        if (prandom_u32() % 2 == 0) {
2177                /* Mutate auth tag (assuming it's at the end of ciphertext) */
2178                flip_random_bit((u8 *)vec->ctext + vec->plen, authsize);
2179        } else {
2180                /* Mutate any part of the ciphertext */
2181                flip_random_bit((u8 *)vec->ctext, vec->clen);
2182        }
2183}
2184
2185/*
2186 * Minimum authentication tag size in bytes at which we assume that we can
2187 * reliably generate inauthentic messages, i.e. not generate an authentic
2188 * message by chance.
2189 */
2190#define MIN_COLLISION_FREE_AUTHSIZE 8
2191
2192static void generate_aead_message(struct aead_request *req,
2193                                  const struct aead_test_suite *suite,
2194                                  struct aead_testvec *vec,
2195                                  bool prefer_inauthentic)
2196{
2197        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2198        const unsigned int ivsize = crypto_aead_ivsize(tfm);
2199        const unsigned int authsize = vec->clen - vec->plen;
2200        const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&
2201                                 (prefer_inauthentic || prandom_u32() % 4 == 0);
2202
2203        /* Generate the AAD. */
2204        generate_random_bytes((u8 *)vec->assoc, vec->alen);
2205        if (suite->aad_iv && vec->alen >= ivsize)
2206                /* Avoid implementation-defined behavior. */
2207                memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);
2208
2209        if (inauthentic && prandom_u32() % 2 == 0) {
2210                /* Generate a random ciphertext. */
2211                generate_random_bytes((u8 *)vec->ctext, vec->clen);
2212        } else {
2213                int i = 0;
2214                struct scatterlist src[2], dst;
2215                u8 iv[MAX_IVLEN];
2216                DECLARE_CRYPTO_WAIT(wait);
2217
2218                /* Generate a random plaintext and encrypt it. */
2219                sg_init_table(src, 2);
2220                if (vec->alen)
2221                        sg_set_buf(&src[i++], vec->assoc, vec->alen);
2222                if (vec->plen) {
2223                        generate_random_bytes((u8 *)vec->ptext, vec->plen);
2224                        sg_set_buf(&src[i++], vec->ptext, vec->plen);
2225                }
2226                sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);
2227                memcpy(iv, vec->iv, ivsize);
2228                aead_request_set_callback(req, 0, crypto_req_done, &wait);
2229                aead_request_set_crypt(req, src, &dst, vec->plen, iv);
2230                aead_request_set_ad(req, vec->alen);
2231                vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req),
2232                                                   &wait);
2233                /* If encryption failed, we're done. */
2234                if (vec->crypt_error != 0)
2235                        return;
2236                memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);
2237                if (!inauthentic)
2238                        return;
2239                /*
2240                 * Mutate the authentic (ciphertext, AAD) pair to get an
2241                 * inauthentic one.
2242                 */
2243                mutate_aead_message(vec, suite->aad_iv, ivsize);
2244        }
2245        vec->novrfy = 1;
2246        if (suite->einval_allowed)
2247                vec->crypt_error = -EINVAL;
2248}
2249
2250/*
2251 * Generate an AEAD test vector 'vec' using the implementation specified by
2252 * 'req'.  The buffers in 'vec' must already be allocated.
2253 *
2254 * If 'prefer_inauthentic' is true, then this function will generate inauthentic
2255 * test vectors (i.e. vectors with 'vec->novrfy=1') more often.
2256 */
2257static void generate_random_aead_testvec(struct aead_request *req,
2258                                         struct aead_testvec *vec,
2259                                         const struct aead_test_suite *suite,
2260                                         unsigned int maxkeysize,
2261                                         unsigned int maxdatasize,
2262                                         char *name, size_t max_namelen,
2263                                         bool prefer_inauthentic)
2264{
2265        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2266        const unsigned int ivsize = crypto_aead_ivsize(tfm);
2267        const unsigned int maxauthsize = crypto_aead_maxauthsize(tfm);
2268        unsigned int authsize;
2269        unsigned int total_len;
2270
2271        /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2272        vec->klen = maxkeysize;
2273        if (prandom_u32() % 4 == 0)
2274                vec->klen = prandom_u32() % (maxkeysize + 1);
2275        generate_random_bytes((u8 *)vec->key, vec->klen);
2276        vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);
2277
2278        /* IV */
2279        generate_random_bytes((u8 *)vec->iv, ivsize);
2280
2281        /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */
2282        authsize = maxauthsize;
2283        if (prandom_u32() % 4 == 0)
2284                authsize = prandom_u32() % (maxauthsize + 1);
2285        if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)
2286                authsize = MIN_COLLISION_FREE_AUTHSIZE;
2287        if (WARN_ON(authsize > maxdatasize))
2288                authsize = maxdatasize;
2289        maxdatasize -= authsize;
2290        vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);
2291
2292        /* AAD, plaintext, and ciphertext lengths */
2293        total_len = generate_random_length(maxdatasize);
2294        if (prandom_u32() % 4 == 0)
2295                vec->alen = 0;
2296        else
2297                vec->alen = generate_random_length(total_len);
2298        vec->plen = total_len - vec->alen;
2299        vec->clen = vec->plen + authsize;
2300
2301        /*
2302         * Generate the AAD, plaintext, and ciphertext.  Not applicable if the
2303         * key or the authentication tag size couldn't be set.
2304         */
2305        vec->novrfy = 0;
2306        vec->crypt_error = 0;
2307        if (vec->setkey_error == 0 && vec->setauthsize_error == 0)
2308                generate_aead_message(req, suite, vec, prefer_inauthentic);
2309        snprintf(name, max_namelen,
2310                 "\"random: alen=%u plen=%u authsize=%u klen=%u novrfy=%d\"",
2311                 vec->alen, vec->plen, authsize, vec->klen, vec->novrfy);
2312}
2313
2314static void try_to_generate_inauthentic_testvec(
2315                                        struct aead_extra_tests_ctx *ctx)
2316{
2317        int i;
2318
2319        for (i = 0; i < 10; i++) {
2320                generate_random_aead_testvec(ctx->req, &ctx->vec,
2321                                             &ctx->test_desc->suite.aead,
2322                                             ctx->maxkeysize, ctx->maxdatasize,
2323                                             ctx->vec_name,
2324                                             sizeof(ctx->vec_name), true);
2325                if (ctx->vec.novrfy)
2326                        return;
2327        }
2328}
2329
2330/*
2331 * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the
2332 * result of an encryption with the key) and verify that decryption fails.
2333 */
2334static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx)
2335{
2336        unsigned int i;
2337        int err;
2338
2339        for (i = 0; i < fuzz_iterations * 8; i++) {
2340                /*
2341                 * Since this part of the tests isn't comparing the
2342                 * implementation to another, there's no point in testing any
2343                 * test vectors other than inauthentic ones (vec.novrfy=1) here.
2344                 *
2345                 * If we're having trouble generating such a test vector, e.g.
2346                 * if the algorithm keeps rejecting the generated keys, don't
2347                 * retry forever; just continue on.
2348                 */
2349                try_to_generate_inauthentic_testvec(ctx);
2350                if (ctx->vec.novrfy) {
2351                        generate_random_testvec_config(&ctx->cfg, ctx->cfgname,
2352                                                       sizeof(ctx->cfgname));
2353                        err = test_aead_vec_cfg(DECRYPT, &ctx->vec,
2354                                                ctx->vec_name, &ctx->cfg,
2355                                                ctx->req, ctx->tsgls);
2356                        if (err)
2357                                return err;
2358                }
2359                cond_resched();
2360        }
2361        return 0;
2362}
2363
2364/*
2365 * Test the AEAD algorithm against the corresponding generic implementation, if
2366 * one is available.
2367 */
2368static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx)
2369{
2370        struct crypto_aead *tfm = ctx->tfm;
2371        const char *algname = crypto_aead_alg(tfm)->base.cra_name;
2372        const char *driver = crypto_aead_driver_name(tfm);
2373        const char *generic_driver = ctx->test_desc->generic_driver;
2374        char _generic_driver[CRYPTO_MAX_ALG_NAME];
2375        struct crypto_aead *generic_tfm = NULL;
2376        struct aead_request *generic_req = NULL;
2377        unsigned int i;
2378        int err;
2379
2380        if (!generic_driver) { /* Use default naming convention? */
2381                err = build_generic_driver_name(algname, _generic_driver);
2382                if (err)
2383                        return err;
2384                generic_driver = _generic_driver;
2385        }
2386
2387        if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2388                return 0;
2389
2390        generic_tfm = crypto_alloc_aead(generic_driver, 0, 0);
2391        if (IS_ERR(generic_tfm)) {
2392                err = PTR_ERR(generic_tfm);
2393                if (err == -ENOENT) {
2394                        pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",
2395                                driver, generic_driver);
2396                        return 0;
2397                }
2398                pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",
2399                       generic_driver, algname, err);
2400                return err;
2401        }
2402
2403        generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
2404        if (!generic_req) {
2405                err = -ENOMEM;
2406                goto out;
2407        }
2408
2409        /* Check the algorithm properties for consistency. */
2410
2411        if (crypto_aead_maxauthsize(tfm) !=
2412            crypto_aead_maxauthsize(generic_tfm)) {
2413                pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",
2414                       driver, crypto_aead_maxauthsize(tfm),
2415                       crypto_aead_maxauthsize(generic_tfm));
2416                err = -EINVAL;
2417                goto out;
2418        }
2419
2420        if (crypto_aead_ivsize(tfm) != crypto_aead_ivsize(generic_tfm)) {
2421                pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",
2422                       driver, crypto_aead_ivsize(tfm),
2423                       crypto_aead_ivsize(generic_tfm));
2424                err = -EINVAL;
2425                goto out;
2426        }
2427
2428        if (crypto_aead_blocksize(tfm) != crypto_aead_blocksize(generic_tfm)) {
2429                pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",
2430                       driver, crypto_aead_blocksize(tfm),
2431                       crypto_aead_blocksize(generic_tfm));
2432                err = -EINVAL;
2433                goto out;
2434        }
2435
2436        /*
2437         * Now generate test vectors using the generic implementation, and test
2438         * the other implementation against them.
2439         */
2440        for (i = 0; i < fuzz_iterations * 8; i++) {
2441                generate_random_aead_testvec(generic_req, &ctx->vec,
2442                                             &ctx->test_desc->suite.aead,
2443                                             ctx->maxkeysize, ctx->maxdatasize,
2444                                             ctx->vec_name,
2445                                             sizeof(ctx->vec_name), false);
2446                generate_random_testvec_config(&ctx->cfg, ctx->cfgname,
2447                                               sizeof(ctx->cfgname));
2448                if (!ctx->vec.novrfy) {
2449                        err = test_aead_vec_cfg(ENCRYPT, &ctx->vec,
2450                                                ctx->vec_name, &ctx->cfg,
2451                                                ctx->req, ctx->tsgls);
2452                        if (err)
2453                                goto out;
2454                }
2455                if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) {
2456                        err = test_aead_vec_cfg(DECRYPT, &ctx->vec,
2457                                                ctx->vec_name, &ctx->cfg,
2458                                                ctx->req, ctx->tsgls);
2459                        if (err)
2460                                goto out;
2461                }
2462                cond_resched();
2463        }
2464        err = 0;
2465out:
2466        crypto_free_aead(generic_tfm);
2467        aead_request_free(generic_req);
2468        return err;
2469}
2470
2471static int test_aead_extra(const struct alg_test_desc *test_desc,
2472                           struct aead_request *req,
2473                           struct cipher_test_sglists *tsgls)
2474{
2475        struct aead_extra_tests_ctx *ctx;
2476        unsigned int i;
2477        int err;
2478
2479        if (noextratests)
2480                return 0;
2481
2482        ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2483        if (!ctx)
2484                return -ENOMEM;
2485        ctx->req = req;
2486        ctx->tfm = crypto_aead_reqtfm(req);
2487        ctx->test_desc = test_desc;
2488        ctx->tsgls = tsgls;
2489        ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2490        ctx->maxkeysize = 0;
2491        for (i = 0; i < test_desc->suite.aead.count; i++)
2492                ctx->maxkeysize = max_t(unsigned int, ctx->maxkeysize,
2493                                        test_desc->suite.aead.vecs[i].klen);
2494
2495        ctx->vec.key = kmalloc(ctx->maxkeysize, GFP_KERNEL);
2496        ctx->vec.iv = kmalloc(crypto_aead_ivsize(ctx->tfm), GFP_KERNEL);
2497        ctx->vec.assoc = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2498        ctx->vec.ptext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2499        ctx->vec.ctext = kmalloc(ctx->maxdatasize, GFP_KERNEL);
2500        if (!ctx->vec.key || !ctx->vec.iv || !ctx->vec.assoc ||
2501            !ctx->vec.ptext || !ctx->vec.ctext) {
2502                err = -ENOMEM;
2503                goto out;
2504        }
2505
2506        err = test_aead_vs_generic_impl(ctx);
2507        if (err)
2508                goto out;
2509
2510        err = test_aead_inauthentic_inputs(ctx);
2511out:
2512        kfree(ctx->vec.key);
2513        kfree(ctx->vec.iv);
2514        kfree(ctx->vec.assoc);
2515        kfree(ctx->vec.ptext);
2516        kfree(ctx->vec.ctext);
2517        kfree(ctx);
2518        return err;
2519}
2520#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2521static int test_aead_extra(const struct alg_test_desc *test_desc,
2522                           struct aead_request *req,
2523                           struct cipher_test_sglists *tsgls)
2524{
2525        return 0;
2526}
2527#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
2528
2529static int test_aead(int enc, const struct aead_test_suite *suite,
2530                     struct aead_request *req,
2531                     struct cipher_test_sglists *tsgls)
2532{
2533        unsigned int i;
2534        int err;
2535
2536        for (i = 0; i < suite->count; i++) {
2537                err = test_aead_vec(enc, &suite->vecs[i], i, req, tsgls);
2538                if (err)
2539                        return err;
2540                cond_resched();
2541        }
2542        return 0;
2543}
2544
2545static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,
2546                         u32 type, u32 mask)
2547{
2548        const struct aead_test_suite *suite = &desc->suite.aead;
2549        struct crypto_aead *tfm;
2550        struct aead_request *req = NULL;
2551        struct cipher_test_sglists *tsgls = NULL;
2552        int err;
2553
2554        if (suite->count <= 0) {
2555                pr_err("alg: aead: empty test suite for %s\n", driver);
2556                return -EINVAL;
2557        }
2558
2559        tfm = crypto_alloc_aead(driver, type, mask);
2560        if (IS_ERR(tfm)) {
2561                pr_err("alg: aead: failed to allocate transform for %s: %ld\n",
2562                       driver, PTR_ERR(tfm));
2563                return PTR_ERR(tfm);
2564        }
2565        driver = crypto_aead_driver_name(tfm);
2566
2567        req = aead_request_alloc(tfm, GFP_KERNEL);
2568        if (!req) {
2569                pr_err("alg: aead: failed to allocate request for %s\n",
2570                       driver);
2571                err = -ENOMEM;
2572                goto out;
2573        }
2574
2575        tsgls = alloc_cipher_test_sglists();
2576        if (!tsgls) {
2577                pr_err("alg: aead: failed to allocate test buffers for %s\n",
2578                       driver);
2579                err = -ENOMEM;
2580                goto out;
2581        }
2582
2583        err = test_aead(ENCRYPT, suite, req, tsgls);
2584        if (err)
2585                goto out;
2586
2587        err = test_aead(DECRYPT, suite, req, tsgls);
2588        if (err)
2589                goto out;
2590
2591        err = test_aead_extra(desc, req, tsgls);
2592out:
2593        free_cipher_test_sglists(tsgls);
2594        aead_request_free(req);
2595        crypto_free_aead(tfm);
2596        return err;
2597}
2598
2599static int test_cipher(struct crypto_cipher *tfm, int enc,
2600                       const struct cipher_testvec *template,
2601                       unsigned int tcount)
2602{
2603        const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm));
2604        unsigned int i, j, k;
2605        char *q;
2606        const char *e;
2607        const char *input, *result;
2608        void *data;
2609        char *xbuf[XBUFSIZE];
2610        int ret = -ENOMEM;
2611
2612        if (testmgr_alloc_buf(xbuf))
2613                goto out_nobuf;
2614
2615        if (enc == ENCRYPT)
2616                e = "encryption";
2617        else
2618                e = "decryption";
2619
2620        j = 0;
2621        for (i = 0; i < tcount; i++) {
2622
2623                if (fips_enabled && template[i].fips_skip)
2624                        continue;
2625
2626                input  = enc ? template[i].ptext : template[i].ctext;
2627                result = enc ? template[i].ctext : template[i].ptext;
2628                j++;
2629
2630                ret = -EINVAL;
2631                if (WARN_ON(template[i].len > PAGE_SIZE))
2632                        goto out;
2633
2634                data = xbuf[0];
2635                memcpy(data, input, template[i].len);
2636
2637                crypto_cipher_clear_flags(tfm, ~0);
2638                if (template[i].wk)
2639                        crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2640
2641                ret = crypto_cipher_setkey(tfm, template[i].key,
2642                                           template[i].klen);
2643                if (ret) {
2644                        if (ret == template[i].setkey_error)
2645                                continue;
2646                        pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",
2647                               algo, j, template[i].setkey_error, ret,
2648                               crypto_cipher_get_flags(tfm));
2649                        goto out;
2650                }
2651                if (template[i].setkey_error) {
2652                        pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",
2653                               algo, j, template[i].setkey_error);
2654                        ret = -EINVAL;
2655                        goto out;
2656                }
2657
2658                for (k = 0; k < template[i].len;
2659                     k += crypto_cipher_blocksize(tfm)) {
2660                        if (enc)
2661                                crypto_cipher_encrypt_one(tfm, data + k,
2662                                                          data + k);
2663                        else
2664                                crypto_cipher_decrypt_one(tfm, data + k,
2665                                                          data + k);
2666                }
2667
2668                q = data;
2669                if (memcmp(q, result, template[i].len)) {
2670                        printk(KERN_ERR "alg: cipher: Test %d failed "
2671                               "on %s for %s\n", j, e, algo);
2672                        hexdump(q, template[i].len);
2673                        ret = -EINVAL;
2674                        goto out;
2675                }
2676        }
2677
2678        ret = 0;
2679
2680out:
2681        testmgr_free_buf(xbuf);
2682out_nobuf:
2683        return ret;
2684}
2685
2686static int test_skcipher_vec_cfg(int enc, const struct cipher_testvec *vec,
2687                                 const char *vec_name,
2688                                 const struct testvec_config *cfg,
2689                                 struct skcipher_request *req,
2690                                 struct cipher_test_sglists *tsgls)
2691{
2692        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2693        const unsigned int alignmask = crypto_skcipher_alignmask(tfm);
2694        const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2695        const char *driver = crypto_skcipher_driver_name(tfm);
2696        const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
2697        const char *op = enc ? "encryption" : "decryption";
2698        DECLARE_CRYPTO_WAIT(wait);
2699        u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];
2700        u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +
2701                 cfg->iv_offset +
2702                 (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);
2703        struct kvec input;
2704        int err;
2705
2706        /* Set the key */
2707        if (vec->wk)
2708                crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2709        else
2710                crypto_skcipher_clear_flags(tfm,
2711                                            CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
2712        err = do_setkey(crypto_skcipher_setkey, tfm, vec->key, vec->klen,
2713                        cfg, alignmask);
2714        if (err) {
2715                if (err == vec->setkey_error)
2716                        return 0;
2717                pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
2718                       driver, vec_name, vec->setkey_error, err,
2719                       crypto_skcipher_get_flags(tfm));
2720                return err;
2721        }
2722        if (vec->setkey_error) {
2723                pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
2724                       driver, vec_name, vec->setkey_error);
2725                return -EINVAL;
2726        }
2727
2728        /* The IV must be copied to a buffer, as the algorithm may modify it */
2729        if (ivsize) {
2730                if (WARN_ON(ivsize > MAX_IVLEN))
2731                        return -EINVAL;
2732                if (vec->generates_iv && !enc)
2733                        memcpy(iv, vec->iv_out, ivsize);
2734                else if (vec->iv)
2735                        memcpy(iv, vec->iv, ivsize);
2736                else
2737                        memset(iv, 0, ivsize);
2738        } else {
2739                if (vec->generates_iv) {
2740                        pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n",
2741                               driver, vec_name);
2742                        return -EINVAL;
2743                }
2744                iv = NULL;
2745        }
2746
2747        /* Build the src/dst scatterlists */
2748        input.iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;
2749        input.iov_len = vec->len;
2750        err = build_cipher_test_sglists(tsgls, cfg, alignmask,
2751                                        vec->len, vec->len, &input, 1);
2752        if (err) {
2753                pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",
2754                       driver, op, vec_name, cfg->name);
2755                return err;
2756        }
2757
2758        /* Do the actual encryption or decryption */
2759        testmgr_poison(req->__ctx, crypto_skcipher_reqsize(tfm));
2760        skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait);
2761        skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,
2762                                   vec->len, iv);
2763        if (cfg->nosimd)
2764                crypto_disable_simd_for_test();
2765        err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
2766        if (cfg->nosimd)
2767                crypto_reenable_simd_for_test();
2768        err = crypto_wait_req(err, &wait);
2769
2770        /* Check that the algorithm didn't overwrite things it shouldn't have */
2771        if (req->cryptlen != vec->len ||
2772            req->iv != iv ||
2773            req->src != tsgls->src.sgl_ptr ||
2774            req->dst != tsgls->dst.sgl_ptr ||
2775            crypto_skcipher_reqtfm(req) != tfm ||
2776            req->base.complete != crypto_req_done ||
2777            req->base.flags != req_flags ||
2778            req->base.data != &wait) {
2779                pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",
2780                       driver, op, vec_name, cfg->name);
2781                if (req->cryptlen != vec->len)
2782                        pr_err("alg: skcipher: changed 'req->cryptlen'\n");
2783                if (req->iv != iv)
2784                        pr_err("alg: skcipher: changed 'req->iv'\n");
2785                if (req->src != tsgls->src.sgl_ptr)
2786                        pr_err("alg: skcipher: changed 'req->src'\n");
2787                if (req->dst != tsgls->dst.sgl_ptr)
2788                        pr_err("alg: skcipher: changed 'req->dst'\n");
2789                if (crypto_skcipher_reqtfm(req) != tfm)
2790                        pr_err("alg: skcipher: changed 'req->base.tfm'\n");
2791                if (req->base.complete != crypto_req_done)
2792                        pr_err("alg: skcipher: changed 'req->base.complete'\n");
2793                if (req->base.flags != req_flags)
2794                        pr_err("alg: skcipher: changed 'req->base.flags'\n");
2795                if (req->base.data != &wait)
2796                        pr_err("alg: skcipher: changed 'req->base.data'\n");
2797                return -EINVAL;
2798        }
2799        if (is_test_sglist_corrupted(&tsgls->src)) {
2800                pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",
2801                       driver, op, vec_name, cfg->name);
2802                return -EINVAL;
2803        }
2804        if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&
2805            is_test_sglist_corrupted(&tsgls->dst)) {
2806                pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",
2807                       driver, op, vec_name, cfg->name);
2808                return -EINVAL;
2809        }
2810
2811        /* Check for success or failure */
2812        if (err) {
2813                if (err == vec->crypt_error)
2814                        return 0;
2815                pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
2816                       driver, op, vec_name, vec->crypt_error, err, cfg->name);
2817                return err;
2818        }
2819        if (vec->crypt_error) {
2820                pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
2821                       driver, op, vec_name, vec->crypt_error, cfg->name);
2822                return -EINVAL;
2823        }
2824
2825        /* Check for the correct output (ciphertext or plaintext) */
2826        err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,
2827                                    vec->len, 0, true);
2828        if (err == -EOVERFLOW) {
2829                pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",
2830                       driver, op, vec_name, cfg->name);
2831                return err;
2832        }
2833        if (err) {
2834                pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
2835                       driver, op, vec_name, cfg->name);
2836                return err;
2837        }
2838
2839        /* If applicable, check that the algorithm generated the correct IV */
2840        if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) {
2841                pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",
2842                       driver, op, vec_name, cfg->name);
2843                hexdump(iv, ivsize);
2844                return -EINVAL;
2845        }
2846
2847        return 0;
2848}
2849
2850static int test_skcipher_vec(int enc, const struct cipher_testvec *vec,
2851                             unsigned int vec_num,
2852                             struct skcipher_request *req,
2853                             struct cipher_test_sglists *tsgls)
2854{
2855        char vec_name[16];
2856        unsigned int i;
2857        int err;
2858
2859        if (fips_enabled && vec->fips_skip)
2860                return 0;
2861
2862        sprintf(vec_name, "%u", vec_num);
2863
2864        for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {
2865                err = test_skcipher_vec_cfg(enc, vec, vec_name,
2866                                            &default_cipher_testvec_configs[i],
2867                                            req, tsgls);
2868                if (err)
2869                        return err;
2870        }
2871
2872#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2873        if (!noextratests) {
2874                struct testvec_config cfg;
2875                char cfgname[TESTVEC_CONFIG_NAMELEN];
2876
2877                for (i = 0; i < fuzz_iterations; i++) {
2878                        generate_random_testvec_config(&cfg, cfgname,
2879                                                       sizeof(cfgname));
2880                        err = test_skcipher_vec_cfg(enc, vec, vec_name,
2881                                                    &cfg, req, tsgls);
2882                        if (err)
2883                                return err;
2884                        cond_resched();
2885                }
2886        }
2887#endif
2888        return 0;
2889}
2890
2891#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
2892/*
2893 * Generate a symmetric cipher test vector from the given implementation.
2894 * Assumes the buffers in 'vec' were already allocated.
2895 */
2896static void generate_random_cipher_testvec(struct skcipher_request *req,
2897                                           struct cipher_testvec *vec,
2898                                           unsigned int maxdatasize,
2899                                           char *name, size_t max_namelen)
2900{
2901        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2902        const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
2903        const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2904        struct scatterlist src, dst;
2905        u8 iv[MAX_IVLEN];
2906        DECLARE_CRYPTO_WAIT(wait);
2907
2908        /* Key: length in [0, maxkeysize], but usually choose maxkeysize */
2909        vec->klen = maxkeysize;
2910        if (prandom_u32() % 4 == 0)
2911                vec->klen = prandom_u32() % (maxkeysize + 1);
2912        generate_random_bytes((u8 *)vec->key, vec->klen);
2913        vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);
2914
2915        /* IV */
2916        generate_random_bytes((u8 *)vec->iv, ivsize);
2917
2918        /* Plaintext */
2919        vec->len = generate_random_length(maxdatasize);
2920        generate_random_bytes((u8 *)vec->ptext, vec->len);
2921
2922        /* If the key couldn't be set, no need to continue to encrypt. */
2923        if (vec->setkey_error)
2924                goto done;
2925
2926        /* Ciphertext */
2927        sg_init_one(&src, vec->ptext, vec->len);
2928        sg_init_one(&dst, vec->ctext, vec->len);
2929        memcpy(iv, vec->iv, ivsize);
2930        skcipher_request_set_callback(req, 0, crypto_req_done, &wait);
2931        skcipher_request_set_crypt(req, &src, &dst, vec->len, iv);
2932        vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
2933        if (vec->crypt_error != 0) {
2934                /*
2935                 * The only acceptable error here is for an invalid length, so
2936                 * skcipher decryption should fail with the same error too.
2937                 * We'll test for this.  But to keep the API usage well-defined,
2938                 * explicitly initialize the ciphertext buffer too.
2939                 */
2940                memset((u8 *)vec->ctext, 0, vec->len);
2941        }
2942done:
2943        snprintf(name, max_namelen, "\"random: len=%u klen=%u\"",
2944                 vec->len, vec->klen);
2945}
2946
2947/*
2948 * Test the skcipher algorithm represented by @req against the corresponding
2949 * generic implementation, if one is available.
2950 */
2951static int test_skcipher_vs_generic_impl(const char *generic_driver,
2952                                         struct skcipher_request *req,
2953                                         struct cipher_test_sglists *tsgls)
2954{
2955        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2956        const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);
2957        const unsigned int ivsize = crypto_skcipher_ivsize(tfm);
2958        const unsigned int blocksize = crypto_skcipher_blocksize(tfm);
2959        const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;
2960        const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;
2961        const char *driver = crypto_skcipher_driver_name(tfm);
2962        char _generic_driver[CRYPTO_MAX_ALG_NAME];
2963        struct crypto_skcipher *generic_tfm = NULL;
2964        struct skcipher_request *generic_req = NULL;
2965        unsigned int i;
2966        struct cipher_testvec vec = { 0 };
2967        char vec_name[64];
2968        struct testvec_config *cfg;
2969        char cfgname[TESTVEC_CONFIG_NAMELEN];
2970        int err;
2971
2972        if (noextratests)
2973                return 0;
2974
2975        /* Keywrap isn't supported here yet as it handles its IV differently. */
2976        if (strncmp(algname, "kw(", 3) == 0)
2977                return 0;
2978
2979        if (!generic_driver) { /* Use default naming convention? */
2980                err = build_generic_driver_name(algname, _generic_driver);
2981                if (err)
2982                        return err;
2983                generic_driver = _generic_driver;
2984        }
2985
2986        if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */
2987                return 0;
2988
2989        generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0);
2990        if (IS_ERR(generic_tfm)) {
2991                err = PTR_ERR(generic_tfm);
2992                if (err == -ENOENT) {
2993                        pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",
2994                                driver, generic_driver);
2995                        return 0;
2996                }
2997                pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",
2998                       generic_driver, algname, err);
2999                return err;
3000        }
3001
3002        cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
3003        if (!cfg) {
3004                err = -ENOMEM;
3005                goto out;
3006        }
3007
3008        generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
3009        if (!generic_req) {
3010                err = -ENOMEM;
3011                goto out;
3012        }
3013
3014        /* Check the algorithm properties for consistency. */
3015
3016        if (crypto_skcipher_min_keysize(tfm) !=
3017            crypto_skcipher_min_keysize(generic_tfm)) {
3018                pr_err("alg: skcipher: min keysize for %s (%u) doesn't match generic impl (%u)\n",
3019                       driver, crypto_skcipher_min_keysize(tfm),
3020                       crypto_skcipher_min_keysize(generic_tfm));
3021                err = -EINVAL;
3022                goto out;
3023        }
3024
3025        if (maxkeysize != crypto_skcipher_max_keysize(generic_tfm)) {
3026                pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",
3027                       driver, maxkeysize,
3028                       crypto_skcipher_max_keysize(generic_tfm));
3029                err = -EINVAL;
3030                goto out;
3031        }
3032
3033        if (ivsize != crypto_skcipher_ivsize(generic_tfm)) {
3034                pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",
3035                       driver, ivsize, crypto_skcipher_ivsize(generic_tfm));
3036                err = -EINVAL;
3037                goto out;
3038        }
3039
3040        if (blocksize != crypto_skcipher_blocksize(generic_tfm)) {
3041                pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",
3042                       driver, blocksize,
3043                       crypto_skcipher_blocksize(generic_tfm));
3044                err = -EINVAL;
3045                goto out;
3046        }
3047
3048        /*
3049         * Now generate test vectors using the generic implementation, and test
3050         * the other implementation against them.
3051         */
3052
3053        vec.key = kmalloc(maxkeysize, GFP_KERNEL);
3054        vec.iv = kmalloc(ivsize, GFP_KERNEL);
3055        vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);
3056        vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);
3057        if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) {
3058                err = -ENOMEM;
3059                goto out;
3060        }
3061
3062        for (i = 0; i < fuzz_iterations * 8; i++) {
3063                generate_random_cipher_testvec(generic_req, &vec, maxdatasize,
3064                                               vec_name, sizeof(vec_name));
3065                generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
3066
3067                err = test_skcipher_vec_cfg(ENCRYPT, &vec, vec_name,
3068                                            cfg, req, tsgls);
3069                if (err)
3070                        goto out;
3071                err = test_skcipher_vec_cfg(DECRYPT, &vec, vec_name,
3072                                            cfg, req, tsgls);
3073                if (err)
3074                        goto out;
3075                cond_resched();
3076        }
3077        err = 0;
3078out:
3079        kfree(cfg);
3080        kfree(vec.key);
3081        kfree(vec.iv);
3082        kfree(vec.ptext);
3083        kfree(vec.ctext);
3084        crypto_free_skcipher(generic_tfm);
3085        skcipher_request_free(generic_req);
3086        return err;
3087}
3088#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
3089static int test_skcipher_vs_generic_impl(const char *generic_driver,
3090                                         struct skcipher_request *req,
3091                                         struct cipher_test_sglists *tsgls)
3092{
3093        return 0;
3094}
3095#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
3096
3097static int test_skcipher(int enc, const struct cipher_test_suite *suite,
3098                         struct skcipher_request *req,
3099                         struct cipher_test_sglists *tsgls)
3100{
3101        unsigned int i;
3102        int err;
3103
3104        for (i = 0; i < suite->count; i++) {
3105                err = test_skcipher_vec(enc, &suite->vecs[i], i, req, tsgls);
3106                if (err)
3107                        return err;
3108                cond_resched();
3109        }
3110        return 0;
3111}
3112
3113static int alg_test_skcipher(const struct alg_test_desc *desc,
3114                             const char *driver, u32 type, u32 mask)
3115{
3116        const struct cipher_test_suite *suite = &desc->suite.cipher;
3117        struct crypto_skcipher *tfm;
3118        struct skcipher_request *req = NULL;
3119        struct cipher_test_sglists *tsgls = NULL;
3120        int err;
3121
3122        if (suite->count <= 0) {
3123                pr_err("alg: skcipher: empty test suite for %s\n", driver);
3124                return -EINVAL;
3125        }
3126
3127        tfm = crypto_alloc_skcipher(driver, type, mask);
3128        if (IS_ERR(tfm)) {
3129                pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",
3130                       driver, PTR_ERR(tfm));
3131                return PTR_ERR(tfm);
3132        }
3133        driver = crypto_skcipher_driver_name(tfm);
3134
3135        req = skcipher_request_alloc(tfm, GFP_KERNEL);
3136        if (!req) {
3137                pr_err("alg: skcipher: failed to allocate request for %s\n",
3138                       driver);
3139                err = -ENOMEM;
3140                goto out;
3141        }
3142
3143        tsgls = alloc_cipher_test_sglists();
3144        if (!tsgls) {
3145                pr_err("alg: skcipher: failed to allocate test buffers for %s\n",
3146                       driver);
3147                err = -ENOMEM;
3148                goto out;
3149        }
3150
3151        err = test_skcipher(ENCRYPT, suite, req, tsgls);
3152        if (err)
3153                goto out;
3154
3155        err = test_skcipher(DECRYPT, suite, req, tsgls);
3156        if (err)
3157                goto out;
3158
3159        err = test_skcipher_vs_generic_impl(desc->generic_driver, req, tsgls);
3160out:
3161        free_cipher_test_sglists(tsgls);
3162        skcipher_request_free(req);
3163        crypto_free_skcipher(tfm);
3164        return err;
3165}
3166
3167static int test_comp(struct crypto_comp *tfm,
3168                     const struct comp_testvec *ctemplate,
3169                     const struct comp_testvec *dtemplate,
3170                     int ctcount, int dtcount)
3171{
3172        const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm));
3173        char *output, *decomp_output;
3174        unsigned int i;
3175        int ret;
3176
3177        output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3178        if (!output)
3179                return -ENOMEM;
3180
3181        decomp_output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3182        if (!decomp_output) {
3183                kfree(output);
3184                return -ENOMEM;
3185        }
3186
3187        for (i = 0; i < ctcount; i++) {
3188                int ilen;
3189                unsigned int dlen = COMP_BUF_SIZE;
3190
3191                memset(output, 0, COMP_BUF_SIZE);
3192                memset(decomp_output, 0, COMP_BUF_SIZE);
3193
3194                ilen = ctemplate[i].inlen;
3195                ret = crypto_comp_compress(tfm, ctemplate[i].input,
3196                                           ilen, output, &dlen);
3197                if (ret) {
3198                        printk(KERN_ERR "alg: comp: compression failed "
3199                               "on test %d for %s: ret=%d\n", i + 1, algo,
3200                               -ret);
3201                        goto out;
3202                }
3203
3204                ilen = dlen;
3205                dlen = COMP_BUF_SIZE;
3206                ret = crypto_comp_decompress(tfm, output,
3207                                             ilen, decomp_output, &dlen);
3208                if (ret) {
3209                        pr_err("alg: comp: compression failed: decompress: on test %d for %s failed: ret=%d\n",
3210                               i + 1, algo, -ret);
3211                        goto out;
3212                }
3213
3214                if (dlen != ctemplate[i].inlen) {
3215                        printk(KERN_ERR "alg: comp: Compression test %d "
3216                               "failed for %s: output len = %d\n", i + 1, algo,
3217                               dlen);
3218                        ret = -EINVAL;
3219                        goto out;
3220                }
3221
3222                if (memcmp(decomp_output, ctemplate[i].input,
3223                           ctemplate[i].inlen)) {
3224                        pr_err("alg: comp: compression failed: output differs: on test %d for %s\n",
3225                               i + 1, algo);
3226                        hexdump(decomp_output, dlen);
3227                        ret = -EINVAL;
3228                        goto out;
3229                }
3230        }
3231
3232        for (i = 0; i < dtcount; i++) {
3233                int ilen;
3234                unsigned int dlen = COMP_BUF_SIZE;
3235
3236                memset(decomp_output, 0, COMP_BUF_SIZE);
3237
3238                ilen = dtemplate[i].inlen;
3239                ret = crypto_comp_decompress(tfm, dtemplate[i].input,
3240                                             ilen, decomp_output, &dlen);
3241                if (ret) {
3242                        printk(KERN_ERR "alg: comp: decompression failed "
3243                               "on test %d for %s: ret=%d\n", i + 1, algo,
3244                               -ret);
3245                        goto out;
3246                }
3247
3248                if (dlen != dtemplate[i].outlen) {
3249                        printk(KERN_ERR "alg: comp: Decompression test %d "
3250                               "failed for %s: output len = %d\n", i + 1, algo,
3251                               dlen);
3252                        ret = -EINVAL;
3253                        goto out;
3254                }
3255
3256                if (memcmp(decomp_output, dtemplate[i].output, dlen)) {
3257                        printk(KERN_ERR "alg: comp: Decompression test %d "
3258                               "failed for %s\n", i + 1, algo);
3259                        hexdump(decomp_output, dlen);
3260                        ret = -EINVAL;
3261                        goto out;
3262                }
3263        }
3264
3265        ret = 0;
3266
3267out:
3268        kfree(decomp_output);
3269        kfree(output);
3270        return ret;
3271}
3272
3273static int test_acomp(struct crypto_acomp *tfm,
3274                              const struct comp_testvec *ctemplate,
3275                      const struct comp_testvec *dtemplate,
3276                      int ctcount, int dtcount)
3277{
3278        const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));
3279        unsigned int i;
3280        char *output, *decomp_out;
3281        int ret;
3282        struct scatterlist src, dst;
3283        struct acomp_req *req;
3284        struct crypto_wait wait;
3285
3286        output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3287        if (!output)
3288                return -ENOMEM;
3289
3290        decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);
3291        if (!decomp_out) {
3292                kfree(output);
3293                return -ENOMEM;
3294        }
3295
3296        for (i = 0; i < ctcount; i++) {
3297                unsigned int dlen = COMP_BUF_SIZE;
3298                int ilen = ctemplate[i].inlen;
3299                void *input_vec;
3300
3301                input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);
3302                if (!input_vec) {
3303                        ret = -ENOMEM;
3304                        goto out;
3305                }
3306
3307                memset(output, 0, dlen);
3308                crypto_init_wait(&wait);
3309                sg_init_one(&src, input_vec, ilen);
3310                sg_init_one(&dst, output, dlen);
3311
3312                req = acomp_request_alloc(tfm);
3313                if (!req) {
3314                        pr_err("alg: acomp: request alloc failed for %s\n",
3315                               algo);
3316                        kfree(input_vec);
3317                        ret = -ENOMEM;
3318                        goto out;
3319                }
3320
3321                acomp_request_set_params(req, &src, &dst, ilen, dlen);
3322                acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3323                                           crypto_req_done, &wait);
3324
3325                ret = crypto_wait_req(crypto_acomp_compress(req), &wait);
3326                if (ret) {
3327                        pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3328                               i + 1, algo, -ret);
3329                        kfree(input_vec);
3330                        acomp_request_free(req);
3331                        goto out;
3332                }
3333
3334                ilen = req->dlen;
3335                dlen = COMP_BUF_SIZE;
3336                sg_init_one(&src, output, ilen);
3337                sg_init_one(&dst, decomp_out, dlen);
3338                crypto_init_wait(&wait);
3339                acomp_request_set_params(req, &src, &dst, ilen, dlen);
3340
3341                ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3342                if (ret) {
3343                        pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",
3344                               i + 1, algo, -ret);
3345                        kfree(input_vec);
3346                        acomp_request_free(req);
3347                        goto out;
3348                }
3349
3350                if (req->dlen != ctemplate[i].inlen) {
3351                        pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",
3352                               i + 1, algo, req->dlen);
3353                        ret = -EINVAL;
3354                        kfree(input_vec);
3355                        acomp_request_free(req);
3356                        goto out;
3357                }
3358
3359                if (memcmp(input_vec, decomp_out, req->dlen)) {
3360                        pr_err("alg: acomp: Compression test %d failed for %s\n",
3361                               i + 1, algo);
3362                        hexdump(output, req->dlen);
3363                        ret = -EINVAL;
3364                        kfree(input_vec);
3365                        acomp_request_free(req);
3366                        goto out;
3367                }
3368
3369                kfree(input_vec);
3370                acomp_request_free(req);
3371        }
3372
3373        for (i = 0; i < dtcount; i++) {
3374                unsigned int dlen = COMP_BUF_SIZE;
3375                int ilen = dtemplate[i].inlen;
3376                void *input_vec;
3377
3378                input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);
3379                if (!input_vec) {
3380                        ret = -ENOMEM;
3381                        goto out;
3382                }
3383
3384                memset(output, 0, dlen);
3385                crypto_init_wait(&wait);
3386                sg_init_one(&src, input_vec, ilen);
3387                sg_init_one(&dst, output, dlen);
3388
3389                req = acomp_request_alloc(tfm);
3390                if (!req) {
3391                        pr_err("alg: acomp: request alloc failed for %s\n",
3392                               algo);
3393                        kfree(input_vec);
3394                        ret = -ENOMEM;
3395                        goto out;
3396                }
3397
3398                acomp_request_set_params(req, &src, &dst, ilen, dlen);
3399                acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3400                                           crypto_req_done, &wait);
3401
3402                ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);
3403                if (ret) {
3404                        pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",
3405                               i + 1, algo, -ret);
3406                        kfree(input_vec);
3407                        acomp_request_free(req);
3408                        goto out;
3409                }
3410
3411                if (req->dlen != dtemplate[i].outlen) {
3412                        pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",
3413                               i + 1, algo, req->dlen);
3414                        ret = -EINVAL;
3415                        kfree(input_vec);
3416                        acomp_request_free(req);
3417                        goto out;
3418                }
3419
3420                if (memcmp(output, dtemplate[i].output, req->dlen)) {
3421                        pr_err("alg: acomp: Decompression test %d failed for %s\n",
3422                               i + 1, algo);
3423                        hexdump(output, req->dlen);
3424                        ret = -EINVAL;
3425                        kfree(input_vec);
3426                        acomp_request_free(req);
3427                        goto out;
3428                }
3429
3430                kfree(input_vec);
3431                acomp_request_free(req);
3432        }
3433
3434        ret = 0;
3435
3436out:
3437        kfree(decomp_out);
3438        kfree(output);
3439        return ret;
3440}
3441
3442static int test_cprng(struct crypto_rng *tfm,
3443                      const struct cprng_testvec *template,
3444                      unsigned int tcount)
3445{
3446        const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm));
3447        int err = 0, i, j, seedsize;
3448        u8 *seed;
3449        char result[32];
3450
3451        seedsize = crypto_rng_seedsize(tfm);
3452
3453        seed = kmalloc(seedsize, GFP_KERNEL);
3454        if (!seed) {
3455                printk(KERN_ERR "alg: cprng: Failed to allocate seed space "
3456                       "for %s\n", algo);
3457                return -ENOMEM;
3458        }
3459
3460        for (i = 0; i < tcount; i++) {
3461                memset(result, 0, 32);
3462
3463                memcpy(seed, template[i].v, template[i].vlen);
3464                memcpy(seed + template[i].vlen, template[i].key,
3465                       template[i].klen);
3466                memcpy(seed + template[i].vlen + template[i].klen,
3467                       template[i].dt, template[i].dtlen);
3468
3469                err = crypto_rng_reset(tfm, seed, seedsize);
3470                if (err) {
3471                        printk(KERN_ERR "alg: cprng: Failed to reset rng "
3472                               "for %s\n", algo);
3473                        goto out;
3474                }
3475
3476                for (j = 0; j < template[i].loops; j++) {
3477                        err = crypto_rng_get_bytes(tfm, result,
3478                                                   template[i].rlen);
3479                        if (err < 0) {
3480                                printk(KERN_ERR "alg: cprng: Failed to obtain "
3481                                       "the correct amount of random data for "
3482                                       "%s (requested %d)\n", algo,
3483                                       template[i].rlen);
3484                                goto out;
3485                        }
3486                }
3487
3488                err = memcmp(result, template[i].result,
3489                             template[i].rlen);
3490                if (err) {
3491                        printk(KERN_ERR "alg: cprng: Test %d failed for %s\n",
3492                               i, algo);
3493                        hexdump(result, template[i].rlen);
3494                        err = -EINVAL;
3495                        goto out;
3496                }
3497        }
3498
3499out:
3500        kfree(seed);
3501        return err;
3502}
3503
3504static int alg_test_cipher(const struct alg_test_desc *desc,
3505                           const char *driver, u32 type, u32 mask)
3506{
3507        const struct cipher_test_suite *suite = &desc->suite.cipher;
3508        struct crypto_cipher *tfm;
3509        int err;
3510
3511        tfm = crypto_alloc_cipher(driver, type, mask);
3512        if (IS_ERR(tfm)) {
3513                printk(KERN_ERR "alg: cipher: Failed to load transform for "
3514                       "%s: %ld\n", driver, PTR_ERR(tfm));
3515                return PTR_ERR(tfm);
3516        }
3517
3518        err = test_cipher(tfm, ENCRYPT, suite->vecs, suite->count);
3519        if (!err)
3520                err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);
3521
3522        crypto_free_cipher(tfm);
3523        return err;
3524}
3525
3526static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
3527                         u32 type, u32 mask)
3528{
3529        struct crypto_comp *comp;
3530        struct crypto_acomp *acomp;
3531        int err;
3532        u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK;
3533
3534        if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) {
3535                acomp = crypto_alloc_acomp(driver, type, mask);
3536                if (IS_ERR(acomp)) {
3537                        pr_err("alg: acomp: Failed to load transform for %s: %ld\n",
3538                               driver, PTR_ERR(acomp));
3539                        return PTR_ERR(acomp);
3540                }
3541                err = test_acomp(acomp, desc->suite.comp.comp.vecs,
3542                                 desc->suite.comp.decomp.vecs,
3543                                 desc->suite.comp.comp.count,
3544                                 desc->suite.comp.decomp.count);
3545                crypto_free_acomp(acomp);
3546        } else {
3547                comp = crypto_alloc_comp(driver, type, mask);
3548                if (IS_ERR(comp)) {
3549                        pr_err("alg: comp: Failed to load transform for %s: %ld\n",
3550                               driver, PTR_ERR(comp));
3551                        return PTR_ERR(comp);
3552                }
3553
3554                err = test_comp(comp, desc->suite.comp.comp.vecs,
3555                                desc->suite.comp.decomp.vecs,
3556                                desc->suite.comp.comp.count,
3557                                desc->suite.comp.decomp.count);
3558
3559                crypto_free_comp(comp);
3560        }
3561        return err;
3562}
3563
3564static int alg_test_crc32c(const struct alg_test_desc *desc,
3565                           const char *driver, u32 type, u32 mask)
3566{
3567        struct crypto_shash *tfm;
3568        __le32 val;
3569        int err;
3570
3571        err = alg_test_hash(desc, driver, type, mask);
3572        if (err)
3573                return err;
3574
3575        tfm = crypto_alloc_shash(driver, type, mask);
3576        if (IS_ERR(tfm)) {
3577                if (PTR_ERR(tfm) == -ENOENT) {
3578                        /*
3579                         * This crc32c implementation is only available through
3580                         * ahash API, not the shash API, so the remaining part
3581                         * of the test is not applicable to it.
3582                         */
3583                        return 0;
3584                }
3585                printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "
3586                       "%ld\n", driver, PTR_ERR(tfm));
3587                return PTR_ERR(tfm);
3588        }
3589        driver = crypto_shash_driver_name(tfm);
3590
3591        do {
3592                SHASH_DESC_ON_STACK(shash, tfm);
3593                u32 *ctx = (u32 *)shash_desc_ctx(shash);
3594
3595                shash->tfm = tfm;
3596
3597                *ctx = 420553207;
3598                err = crypto_shash_final(shash, (u8 *)&val);
3599                if (err) {
3600                        printk(KERN_ERR "alg: crc32c: Operation failed for "
3601                               "%s: %d\n", driver, err);
3602                        break;
3603                }
3604
3605                if (val != cpu_to_le32(~420553207)) {
3606                        pr_err("alg: crc32c: Test failed for %s: %u\n",
3607                               driver, le32_to_cpu(val));
3608                        err = -EINVAL;
3609                }
3610        } while (0);
3611
3612        crypto_free_shash(tfm);
3613
3614        return err;
3615}
3616
3617static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,
3618                          u32 type, u32 mask)
3619{
3620        struct crypto_rng *rng;
3621        int err;
3622
3623        rng = crypto_alloc_rng(driver, type, mask);
3624        if (IS_ERR(rng)) {
3625                printk(KERN_ERR "alg: cprng: Failed to load transform for %s: "
3626                       "%ld\n", driver, PTR_ERR(rng));
3627                return PTR_ERR(rng);
3628        }
3629
3630        err = test_cprng(rng, desc->suite.cprng.vecs, desc->suite.cprng.count);
3631
3632        crypto_free_rng(rng);
3633
3634        return err;
3635}
3636
3637
3638static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
3639                          const char *driver, u32 type, u32 mask)
3640{
3641        int ret = -EAGAIN;
3642        struct crypto_rng *drng;
3643        struct drbg_test_data test_data;
3644        struct drbg_string addtl, pers, testentropy;
3645        unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);
3646
3647        if (!buf)
3648                return -ENOMEM;
3649
3650        drng = crypto_alloc_rng(driver, type, mask);
3651        if (IS_ERR(drng)) {
3652                printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "
3653                       "%s\n", driver);
3654                kfree_sensitive(buf);
3655                return -ENOMEM;
3656        }
3657
3658        test_data.testentropy = &testentropy;
3659        drbg_string_fill(&testentropy, test->entropy, test->entropylen);
3660        drbg_string_fill(&pers, test->pers, test->perslen);
3661        ret = crypto_drbg_reset_test(drng, &pers, &test_data);
3662        if (ret) {
3663                printk(KERN_ERR "alg: drbg: Failed to reset rng\n");
3664                goto outbuf;
3665        }
3666
3667        drbg_string_fill(&addtl, test->addtla, test->addtllen);
3668        if (pr) {
3669                drbg_string_fill(&testentropy, test->entpra, test->entprlen);
3670                ret = crypto_drbg_get_bytes_addtl_test(drng,
3671                        buf, test->expectedlen, &addtl, &test_data);
3672        } else {
3673                ret = crypto_drbg_get_bytes_addtl(drng,
3674                        buf, test->expectedlen, &addtl);
3675        }
3676        if (ret < 0) {
3677                printk(KERN_ERR "alg: drbg: could not obtain random data for "
3678                       "driver %s\n", driver);
3679                goto outbuf;
3680        }
3681
3682        drbg_string_fill(&addtl, test->addtlb, test->addtllen);
3683        if (pr) {
3684                drbg_string_fill(&testentropy, test->entprb, test->entprlen);
3685                ret = crypto_drbg_get_bytes_addtl_test(drng,
3686                        buf, test->expectedlen, &addtl, &test_data);
3687        } else {
3688                ret = crypto_drbg_get_bytes_addtl(drng,
3689                        buf, test->expectedlen, &addtl);
3690        }
3691        if (ret < 0) {
3692                printk(KERN_ERR "alg: drbg: could not obtain random data for "
3693                       "driver %s\n", driver);
3694                goto outbuf;
3695        }
3696
3697        ret = memcmp(test->expected, buf, test->expectedlen);
3698
3699outbuf:
3700        crypto_free_rng(drng);
3701        kfree_sensitive(buf);
3702        return ret;
3703}
3704
3705
3706static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,
3707                         u32 type, u32 mask)
3708{
3709        int err = 0;
3710        int pr = 0;
3711        int i = 0;
3712        const struct drbg_testvec *template = desc->suite.drbg.vecs;
3713        unsigned int tcount = desc->suite.drbg.count;
3714
3715        if (0 == memcmp(driver, "drbg_pr_", 8))
3716                pr = 1;
3717
3718        for (i = 0; i < tcount; i++) {
3719                err = drbg_cavs_test(&template[i], pr, driver, type, mask);
3720                if (err) {
3721                        printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",
3722                               i, driver);
3723                        err = -EINVAL;
3724                        break;
3725                }
3726        }
3727        return err;
3728
3729}
3730
3731static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
3732                       const char *alg)
3733{
3734        struct kpp_request *req;
3735        void *input_buf = NULL;
3736        void *output_buf = NULL;
3737        void *a_public = NULL;
3738        void *a_ss = NULL;
3739        void *shared_secret = NULL;
3740        struct crypto_wait wait;
3741        unsigned int out_len_max;
3742        int err = -ENOMEM;
3743        struct scatterlist src, dst;
3744
3745        req = kpp_request_alloc(tfm, GFP_KERNEL);
3746        if (!req)
3747                return err;
3748
3749        crypto_init_wait(&wait);
3750
3751        err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size);
3752        if (err < 0)
3753                goto free_req;
3754
3755        out_len_max = crypto_kpp_maxsize(tfm);
3756        output_buf = kzalloc(out_len_max, GFP_KERNEL);
3757        if (!output_buf) {
3758                err = -ENOMEM;
3759                goto free_req;
3760        }
3761
3762        /* Use appropriate parameter as base */
3763        kpp_request_set_input(req, NULL, 0);
3764        sg_init_one(&dst, output_buf, out_len_max);
3765        kpp_request_set_output(req, &dst, out_len_max);
3766        kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3767                                 crypto_req_done, &wait);
3768
3769        /* Compute party A's public key */
3770        err = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait);
3771        if (err) {
3772                pr_err("alg: %s: Party A: generate public key test failed. err %d\n",
3773                       alg, err);
3774                goto free_output;
3775        }
3776
3777        if (vec->genkey) {
3778                /* Save party A's public key */
3779                a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);
3780                if (!a_public) {
3781                        err = -ENOMEM;
3782                        goto free_output;
3783                }
3784        } else {
3785                /* Verify calculated public key */
3786                if (memcmp(vec->expected_a_public, sg_virt(req->dst),
3787                           vec->expected_a_public_size)) {
3788                        pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",
3789                               alg);
3790                        err = -EINVAL;
3791                        goto free_output;
3792                }
3793        }
3794
3795        /* Calculate shared secret key by using counter part (b) public key. */
3796        input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);
3797        if (!input_buf) {
3798                err = -ENOMEM;
3799                goto free_output;
3800        }
3801
3802        sg_init_one(&src, input_buf, vec->b_public_size);
3803        sg_init_one(&dst, output_buf, out_len_max);
3804        kpp_request_set_input(req, &src, vec->b_public_size);
3805        kpp_request_set_output(req, &dst, out_len_max);
3806        kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3807                                 crypto_req_done, &wait);
3808        err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait);
3809        if (err) {
3810                pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",
3811                       alg, err);
3812                goto free_all;
3813        }
3814
3815        if (vec->genkey) {
3816                /* Save the shared secret obtained by party A */
3817                a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);
3818                if (!a_ss) {
3819                        err = -ENOMEM;
3820                        goto free_all;
3821                }
3822
3823                /*
3824                 * Calculate party B's shared secret by using party A's
3825                 * public key.
3826                 */
3827                err = crypto_kpp_set_secret(tfm, vec->b_secret,
3828                                            vec->b_secret_size);
3829                if (err < 0)
3830                        goto free_all;
3831
3832                sg_init_one(&src, a_public, vec->expected_a_public_size);
3833                sg_init_one(&dst, output_buf, out_len_max);
3834                kpp_request_set_input(req, &src, vec->expected_a_public_size);
3835                kpp_request_set_output(req, &dst, out_len_max);
3836                kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
3837                                         crypto_req_done, &wait);
3838                err = crypto_wait_req(crypto_kpp_compute_shared_secret(req),
3839                                      &wait);
3840                if (err) {
3841                        pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",
3842                               alg, err);
3843                        goto free_all;
3844                }
3845
3846                shared_secret = a_ss;
3847        } else {
3848                shared_secret = (void *)vec->expected_ss;
3849        }
3850
3851        /*
3852         * verify shared secret from which the user will derive
3853         * secret key by executing whatever hash it has chosen
3854         */
3855        if (memcmp(shared_secret, sg_virt(req->dst),
3856                   vec->expected_ss_size)) {
3857                pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
3858                       alg);
3859                err = -EINVAL;
3860        }
3861
3862free_all:
3863        kfree(a_ss);
3864        kfree(input_buf);
3865free_output:
3866        kfree(a_public);
3867        kfree(output_buf);
3868free_req:
3869        kpp_request_free(req);
3870        return err;
3871}
3872
3873static int test_kpp(struct crypto_kpp *tfm, const char *alg,
3874                    const struct kpp_testvec *vecs, unsigned int tcount)
3875{
3876        int ret, i;
3877
3878        for (i = 0; i < tcount; i++) {
3879                ret = do_test_kpp(tfm, vecs++, alg);
3880                if (ret) {
3881                        pr_err("alg: %s: test failed on vector %d, err=%d\n",
3882                               alg, i + 1, ret);
3883                        return ret;
3884                }
3885        }
3886        return 0;
3887}
3888
3889static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,
3890                        u32 type, u32 mask)
3891{
3892        struct crypto_kpp *tfm;
3893        int err = 0;
3894
3895        tfm = crypto_alloc_kpp(driver, type, mask);
3896        if (IS_ERR(tfm)) {
3897                pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",
3898                       driver, PTR_ERR(tfm));
3899                return PTR_ERR(tfm);
3900        }
3901        if (desc->suite.kpp.vecs)
3902                err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs,
3903                               desc->suite.kpp.count);
3904
3905        crypto_free_kpp(tfm);
3906        return err;
3907}
3908
3909static u8 *test_pack_u32(u8 *dst, u32 val)
3910{
3911        memcpy(dst, &val, sizeof(val));
3912        return dst + sizeof(val);
3913}
3914
3915static int test_akcipher_one(struct crypto_akcipher *tfm,
3916                             const struct akcipher_testvec *vecs)
3917{
3918        char *xbuf[XBUFSIZE];
3919        struct akcipher_request *req;
3920        void *outbuf_enc = NULL;
3921        void *outbuf_dec = NULL;
3922        struct crypto_wait wait;
3923        unsigned int out_len_max, out_len = 0;
3924        int err = -ENOMEM;
3925        struct scatterlist src, dst, src_tab[3];
3926        const char *m, *c;
3927        unsigned int m_size, c_size;
3928        const char *op;
3929        u8 *key, *ptr;
3930
3931        if (testmgr_alloc_buf(xbuf))
3932                return err;
3933
3934        req = akcipher_request_alloc(tfm, GFP_KERNEL);
3935        if (!req)
3936                goto free_xbuf;
3937
3938        crypto_init_wait(&wait);
3939
3940        key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len,
3941                      GFP_KERNEL);
3942        if (!key)
3943                goto free_req;
3944        memcpy(key, vecs->key, vecs->key_len);
3945        ptr = key + vecs->key_len;
3946        ptr = test_pack_u32(ptr, vecs->algo);
3947        ptr = test_pack_u32(ptr, vecs->param_len);
3948        memcpy(ptr, vecs->params, vecs->param_len);
3949
3950        if (vecs->public_key_vec)
3951                err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len);
3952        else
3953                err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len);
3954        if (err)
3955                goto free_key;
3956
3957        /*
3958         * First run test which do not require a private key, such as
3959         * encrypt or verify.
3960         */
3961        err = -ENOMEM;
3962        out_len_max = crypto_akcipher_maxsize(tfm);
3963        outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
3964        if (!outbuf_enc)
3965                goto free_key;
3966
3967        if (!vecs->siggen_sigver_test) {
3968                m = vecs->m;
3969                m_size = vecs->m_size;
3970                c = vecs->c;
3971                c_size = vecs->c_size;
3972                op = "encrypt";
3973        } else {
3974                /* Swap args so we could keep plaintext (digest)
3975                 * in vecs->m, and cooked signature in vecs->c.
3976                 */
3977                m = vecs->c; /* signature */
3978                m_size = vecs->c_size;
3979                c = vecs->m; /* digest */
3980                c_size = vecs->m_size;
3981                op = "verify";
3982        }
3983
3984        err = -E2BIG;
3985        if (WARN_ON(m_size > PAGE_SIZE))
3986                goto free_all;
3987        memcpy(xbuf[0], m, m_size);
3988
3989        sg_init_table(src_tab, 3);
3990        sg_set_buf(&src_tab[0], xbuf[0], 8);
3991        sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8);
3992        if (vecs->siggen_sigver_test) {
3993                if (WARN_ON(c_size > PAGE_SIZE))
3994                        goto free_all;
3995                memcpy(xbuf[1], c, c_size);
3996                sg_set_buf(&src_tab[2], xbuf[1], c_size);
3997                akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size);
3998        } else {
3999                sg_init_one(&dst, outbuf_enc, out_len_max);
4000                akcipher_request_set_crypt(req, src_tab, &dst, m_size,
4001                                           out_len_max);
4002        }
4003        akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
4004                                      crypto_req_done, &wait);
4005
4006        err = crypto_wait_req(vecs->siggen_sigver_test ?
4007                              /* Run asymmetric signature verification */
4008                              crypto_akcipher_verify(req) :
4009                              /* Run asymmetric encrypt */
4010                              crypto_akcipher_encrypt(req), &wait);
4011        if (err) {
4012                pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
4013                goto free_all;
4014        }
4015        if (!vecs->siggen_sigver_test && c) {
4016                if (req->dst_len != c_size) {
4017                        pr_err("alg: akcipher: %s test failed. Invalid output len\n",
4018                               op);
4019                        err = -EINVAL;
4020                        goto free_all;
4021                }
4022                /* verify that encrypted message is equal to expected */
4023                if (memcmp(c, outbuf_enc, c_size) != 0) {
4024                        pr_err("alg: akcipher: %s test failed. Invalid output\n",
4025                               op);
4026                        hexdump(outbuf_enc, c_size);
4027                        err = -EINVAL;
4028                        goto free_all;
4029                }
4030        }
4031
4032        /*
4033         * Don't invoke (decrypt or sign) test which require a private key
4034         * for vectors with only a public key.
4035         */
4036        if (vecs->public_key_vec) {
4037                err = 0;
4038                goto free_all;
4039        }
4040        outbuf_dec = kzalloc(out_len_max, GFP_KERNEL);
4041        if (!outbuf_dec) {
4042                err = -ENOMEM;
4043                goto free_all;
4044        }
4045
4046        if (!vecs->siggen_sigver_test && !c) {
4047                c = outbuf_enc;
4048                c_size = req->dst_len;
4049        }
4050
4051        err = -E2BIG;
4052        op = vecs->siggen_sigver_test ? "sign" : "decrypt";
4053        if (WARN_ON(c_size > PAGE_SIZE))
4054                goto free_all;
4055        memcpy(xbuf[0], c, c_size);
4056
4057        sg_init_one(&src, xbuf[0], c_size);
4058        sg_init_one(&dst, outbuf_dec, out_len_max);
4059        crypto_init_wait(&wait);
4060        akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max);
4061
4062        err = crypto_wait_req(vecs->siggen_sigver_test ?
4063                              /* Run asymmetric signature generation */
4064                              crypto_akcipher_sign(req) :
4065                              /* Run asymmetric decrypt */
4066                              crypto_akcipher_decrypt(req), &wait);
4067        if (err) {
4068                pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
4069                goto free_all;
4070        }
4071        out_len = req->dst_len;
4072        if (out_len < m_size) {
4073                pr_err("alg: akcipher: %s test failed. Invalid output len %u\n",
4074                       op, out_len);
4075                err = -EINVAL;
4076                goto free_all;
4077        }
4078        /* verify that decrypted message is equal to the original msg */
4079        if (memchr_inv(outbuf_dec, 0, out_len - m_size) ||
4080            memcmp(m, outbuf_dec + out_len - m_size, m_size)) {
4081                pr_err("alg: akcipher: %s test failed. Invalid output\n", op);
4082                hexdump(outbuf_dec, out_len);
4083                err = -EINVAL;
4084        }
4085free_all:
4086        kfree(outbuf_dec);
4087        kfree(outbuf_enc);
4088free_key:
4089        kfree(key);
4090free_req:
4091        akcipher_request_free(req);
4092free_xbuf:
4093        testmgr_free_buf(xbuf);
4094        return err;
4095}
4096
4097static int test_akcipher(struct crypto_akcipher *tfm, const char *alg,
4098                         const struct akcipher_testvec *vecs,
4099                         unsigned int tcount)
4100{
4101        const char *algo =
4102                crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm));
4103        int ret, i;
4104
4105        for (i = 0; i < tcount; i++) {
4106                ret = test_akcipher_one(tfm, vecs++);
4107                if (!ret)
4108                        continue;
4109
4110                pr_err("alg: akcipher: test %d failed for %s, err=%d\n",
4111                       i + 1, algo, ret);
4112                return ret;
4113        }
4114        return 0;
4115}
4116
4117static int alg_test_akcipher(const struct alg_test_desc *desc,
4118                             const char *driver, u32 type, u32 mask)
4119{
4120        struct crypto_akcipher *tfm;
4121        int err = 0;
4122
4123        tfm = crypto_alloc_akcipher(driver, type, mask);
4124        if (IS_ERR(tfm)) {
4125                pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",
4126                       driver, PTR_ERR(tfm));
4127                return PTR_ERR(tfm);
4128        }
4129        if (desc->suite.akcipher.vecs)
4130                err = test_akcipher(tfm, desc->alg, desc->suite.akcipher.vecs,
4131                                    desc->suite.akcipher.count);
4132
4133        crypto_free_akcipher(tfm);
4134        return err;
4135}
4136
4137static int alg_test_null(const struct alg_test_desc *desc,
4138                             const char *driver, u32 type, u32 mask)
4139{
4140        return 0;
4141}
4142
4143#define ____VECS(tv)    .vecs = tv, .count = ARRAY_SIZE(tv)
4144#define __VECS(tv)      { ____VECS(tv) }
4145
4146/* Please keep this list sorted by algorithm name. */
4147static const struct alg_test_desc alg_test_descs[] = {
4148        {
4149                .alg = "adiantum(xchacha12,aes)",
4150                .generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)",
4151                .test = alg_test_skcipher,
4152                .suite = {
4153                        .cipher = __VECS(adiantum_xchacha12_aes_tv_template)
4154                },
4155        }, {
4156                .alg = "adiantum(xchacha20,aes)",
4157                .generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)",
4158                .test = alg_test_skcipher,
4159                .suite = {
4160                        .cipher = __VECS(adiantum_xchacha20_aes_tv_template)
4161                },
4162        }, {
4163                .alg = "aegis128",
4164                .test = alg_test_aead,
4165                .suite = {
4166                        .aead = __VECS(aegis128_tv_template)
4167                }
4168        }, {
4169                .alg = "ansi_cprng",
4170                .test = alg_test_cprng,
4171                .suite = {
4172                        .cprng = __VECS(ansi_cprng_aes_tv_template)
4173                }
4174        }, {
4175                .alg = "authenc(hmac(md5),ecb(cipher_null))",
4176                .test = alg_test_aead,
4177                .suite = {
4178                        .aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)
4179                }
4180        }, {
4181                .alg = "authenc(hmac(sha1),cbc(aes))",
4182                .test = alg_test_aead,
4183                .fips_allowed = 1,
4184                .suite = {
4185                        .aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
4186                }
4187        }, {
4188                .alg = "authenc(hmac(sha1),cbc(des))",
4189                .test = alg_test_aead,
4190                .suite = {
4191                        .aead = __VECS(hmac_sha1_des_cbc_tv_temp)
4192                }
4193        }, {
4194                .alg = "authenc(hmac(sha1),cbc(des3_ede))",
4195                .test = alg_test_aead,
4196                .fips_allowed = 1,
4197                .suite = {
4198                        .aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
4199                }
4200        }, {
4201                .alg = "authenc(hmac(sha1),ctr(aes))",
4202                .test = alg_test_null,
4203                .fips_allowed = 1,
4204        }, {
4205                .alg = "authenc(hmac(sha1),ecb(cipher_null))",
4206                .test = alg_test_aead,
4207                .suite = {
4208                        .aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)
4209                }
4210        }, {
4211                .alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4212                .test = alg_test_null,
4213                .fips_allowed = 1,
4214        }, {
4215                .alg = "authenc(hmac(sha224),cbc(des))",
4216                .test = alg_test_aead,
4217                .suite = {
4218                        .aead = __VECS(hmac_sha224_des_cbc_tv_temp)
4219                }
4220        }, {
4221                .alg = "authenc(hmac(sha224),cbc(des3_ede))",
4222                .test = alg_test_aead,
4223                .fips_allowed = 1,
4224                .suite = {
4225                        .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
4226                }
4227        }, {
4228                .alg = "authenc(hmac(sha256),cbc(aes))",
4229                .test = alg_test_aead,
4230                .fips_allowed = 1,
4231                .suite = {
4232                        .aead = __VECS(hmac_sha256_aes_cbc_tv_temp)
4233                }
4234        }, {
4235                .alg = "authenc(hmac(sha256),cbc(des))",
4236                .test = alg_test_aead,
4237                .suite = {
4238                        .aead = __VECS(hmac_sha256_des_cbc_tv_temp)
4239                }
4240        }, {
4241                .alg = "authenc(hmac(sha256),cbc(des3_ede))",
4242                .test = alg_test_aead,
4243                .fips_allowed = 1,
4244                .suite = {
4245                        .aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
4246                }
4247        }, {
4248                .alg = "authenc(hmac(sha256),ctr(aes))",
4249                .test = alg_test_null,
4250                .fips_allowed = 1,
4251        }, {
4252                .alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4253                .test = alg_test_null,
4254                .fips_allowed = 1,
4255        }, {
4256                .alg = "authenc(hmac(sha384),cbc(des))",
4257                .test = alg_test_aead,
4258                .suite = {
4259                        .aead = __VECS(hmac_sha384_des_cbc_tv_temp)
4260                }
4261        }, {
4262                .alg = "authenc(hmac(sha384),cbc(des3_ede))",
4263                .test = alg_test_aead,
4264                .fips_allowed = 1,
4265                .suite = {
4266                        .aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
4267                }
4268        }, {
4269                .alg = "authenc(hmac(sha384),ctr(aes))",
4270                .test = alg_test_null,
4271                .fips_allowed = 1,
4272        }, {
4273                .alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4274                .test = alg_test_null,
4275                .fips_allowed = 1,
4276        }, {
4277                .alg = "authenc(hmac(sha512),cbc(aes))",
4278                .fips_allowed = 1,
4279                .test = alg_test_aead,
4280                .suite = {
4281                        .aead = __VECS(hmac_sha512_aes_cbc_tv_temp)
4282                }
4283        }, {
4284                .alg = "authenc(hmac(sha512),cbc(des))",
4285                .test = alg_test_aead,
4286                .suite = {
4287                        .aead = __VECS(hmac_sha512_des_cbc_tv_temp)
4288                }
4289        }, {
4290                .alg = "authenc(hmac(sha512),cbc(des3_ede))",
4291                .test = alg_test_aead,
4292                .fips_allowed = 1,
4293                .suite = {
4294                        .aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
4295                }
4296        }, {
4297                .alg = "authenc(hmac(sha512),ctr(aes))",
4298                .test = alg_test_null,
4299                .fips_allowed = 1,
4300        }, {
4301                .alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4302                .test = alg_test_null,
4303                .fips_allowed = 1,
4304        }, {
4305                .alg = "blake2b-160",
4306                .test = alg_test_hash,
4307                .fips_allowed = 0,
4308                .suite = {
4309                        .hash = __VECS(blake2b_160_tv_template)
4310                }
4311        }, {
4312                .alg = "blake2b-256",
4313                .test = alg_test_hash,
4314                .fips_allowed = 0,
4315                .suite = {
4316                        .hash = __VECS(blake2b_256_tv_template)
4317                }
4318        }, {
4319                .alg = "blake2b-384",
4320                .test = alg_test_hash,
4321                .fips_allowed = 0,
4322                .suite = {
4323                        .hash = __VECS(blake2b_384_tv_template)
4324                }
4325        }, {
4326                .alg = "blake2b-512",
4327                .test = alg_test_hash,
4328                .fips_allowed = 0,
4329                .suite = {
4330                        .hash = __VECS(blake2b_512_tv_template)
4331                }
4332        }, {
4333                .alg = "blake2s-128",
4334                .test = alg_test_hash,
4335                .suite = {
4336                        .hash = __VECS(blakes2s_128_tv_template)
4337                }
4338        }, {
4339                .alg = "blake2s-160",
4340                .test = alg_test_hash,
4341                .suite = {
4342                        .hash = __VECS(blakes2s_160_tv_template)
4343                }
4344        }, {
4345                .alg = "blake2s-224",
4346                .test = alg_test_hash,
4347                .suite = {
4348                        .hash = __VECS(blakes2s_224_tv_template)
4349                }
4350        }, {
4351                .alg = "blake2s-256",
4352                .test = alg_test_hash,
4353                .suite = {
4354                        .hash = __VECS(blakes2s_256_tv_template)
4355                }
4356        }, {
4357                .alg = "cbc(aes)",
4358                .test = alg_test_skcipher,
4359                .fips_allowed = 1,
4360                .suite = {
4361                        .cipher = __VECS(aes_cbc_tv_template)
4362                },
4363        }, {
4364                .alg = "cbc(anubis)",
4365                .test = alg_test_skcipher,
4366                .suite = {
4367                        .cipher = __VECS(anubis_cbc_tv_template)
4368                },
4369        }, {
4370                .alg = "cbc(blowfish)",
4371                .test = alg_test_skcipher,
4372                .suite = {
4373                        .cipher = __VECS(bf_cbc_tv_template)
4374                },
4375        }, {
4376                .alg = "cbc(camellia)",
4377                .test = alg_test_skcipher,
4378                .suite = {
4379                        .cipher = __VECS(camellia_cbc_tv_template)
4380                },
4381        }, {
4382                .alg = "cbc(cast5)",
4383                .test = alg_test_skcipher,
4384                .suite = {
4385                        .cipher = __VECS(cast5_cbc_tv_template)
4386                },
4387        }, {
4388                .alg = "cbc(cast6)",
4389                .test = alg_test_skcipher,
4390                .suite = {
4391                        .cipher = __VECS(cast6_cbc_tv_template)
4392                },
4393        }, {
4394                .alg = "cbc(des)",
4395                .test = alg_test_skcipher,
4396                .suite = {
4397                        .cipher = __VECS(des_cbc_tv_template)
4398                },
4399        }, {
4400                .alg = "cbc(des3_ede)",
4401                .test = alg_test_skcipher,
4402                .fips_allowed = 1,
4403                .suite = {
4404                        .cipher = __VECS(des3_ede_cbc_tv_template)
4405                },
4406        }, {
4407                /* Same as cbc(aes) except the key is stored in
4408                 * hardware secure memory which we reference by index
4409                 */
4410                .alg = "cbc(paes)",
4411                .test = alg_test_null,
4412                .fips_allowed = 1,
4413        }, {
4414                /* Same as cbc(sm4) except the key is stored in
4415                 * hardware secure memory which we reference by index
4416                 */
4417                .alg = "cbc(psm4)",
4418                .test = alg_test_null,
4419        }, {
4420                .alg = "cbc(serpent)",
4421                .test = alg_test_skcipher,
4422                .suite = {
4423                        .cipher = __VECS(serpent_cbc_tv_template)
4424                },
4425        }, {
4426                .alg = "cbc(sm4)",
4427                .test = alg_test_skcipher,
4428                .suite = {
4429                        .cipher = __VECS(sm4_cbc_tv_template)
4430                }
4431        }, {
4432                .alg = "cbc(twofish)",
4433                .test = alg_test_skcipher,
4434                .suite = {
4435                        .cipher = __VECS(tf_cbc_tv_template)
4436                },
4437        }, {
4438#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4439                .alg = "cbc-paes-s390",
4440                .fips_allowed = 1,
4441                .test = alg_test_skcipher,
4442                .suite = {
4443                        .cipher = __VECS(aes_cbc_tv_template)
4444                }
4445        }, {
4446#endif
4447                .alg = "cbcmac(aes)",
4448                .fips_allowed = 1,
4449                .test = alg_test_hash,
4450                .suite = {
4451                        .hash = __VECS(aes_cbcmac_tv_template)
4452                }
4453        }, {
4454                .alg = "ccm(aes)",
4455                .generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",
4456                .test = alg_test_aead,
4457                .fips_allowed = 1,
4458                .suite = {
4459                        .aead = {
4460                                ____VECS(aes_ccm_tv_template),
4461                                .einval_allowed = 1,
4462                        }
4463                }
4464        }, {
4465                .alg = "cfb(aes)",
4466                .test = alg_test_skcipher,
4467                .fips_allowed = 1,
4468                .suite = {
4469                        .cipher = __VECS(aes_cfb_tv_template)
4470                },
4471        }, {
4472                .alg = "cfb(sm4)",
4473                .test = alg_test_skcipher,
4474                .suite = {
4475                        .cipher = __VECS(sm4_cfb_tv_template)
4476                }
4477        }, {
4478                .alg = "chacha20",
4479                .test = alg_test_skcipher,
4480                .suite = {
4481                        .cipher = __VECS(chacha20_tv_template)
4482                },
4483        }, {
4484                .alg = "cmac(aes)",
4485                .fips_allowed = 1,
4486                .test = alg_test_hash,
4487                .suite = {
4488                        .hash = __VECS(aes_cmac128_tv_template)
4489                }
4490        }, {
4491                .alg = "cmac(des3_ede)",
4492                .fips_allowed = 1,
4493                .test = alg_test_hash,
4494                .suite = {
4495                        .hash = __VECS(des3_ede_cmac64_tv_template)
4496                }
4497        }, {
4498                .alg = "compress_null",
4499                .test = alg_test_null,
4500        }, {
4501                .alg = "crc32",
4502                .test = alg_test_hash,
4503                .fips_allowed = 1,
4504                .suite = {
4505                        .hash = __VECS(crc32_tv_template)
4506                }
4507        }, {
4508                .alg = "crc32c",
4509                .test = alg_test_crc32c,
4510                .fips_allowed = 1,
4511                .suite = {
4512                        .hash = __VECS(crc32c_tv_template)
4513                }
4514        }, {
4515                .alg = "crct10dif",
4516                .test = alg_test_hash,
4517                .fips_allowed = 1,
4518                .suite = {
4519                        .hash = __VECS(crct10dif_tv_template)
4520                }
4521        }, {
4522                .alg = "ctr(aes)",
4523                .test = alg_test_skcipher,
4524                .fips_allowed = 1,
4525                .suite = {
4526                        .cipher = __VECS(aes_ctr_tv_template)
4527                }
4528        }, {
4529                .alg = "ctr(blowfish)",
4530                .test = alg_test_skcipher,
4531                .suite = {
4532                        .cipher = __VECS(bf_ctr_tv_template)
4533                }
4534        }, {
4535                .alg = "ctr(camellia)",
4536                .test = alg_test_skcipher,
4537                .suite = {
4538                        .cipher = __VECS(camellia_ctr_tv_template)
4539                }
4540        }, {
4541                .alg = "ctr(cast5)",
4542                .test = alg_test_skcipher,
4543                .suite = {
4544                        .cipher = __VECS(cast5_ctr_tv_template)
4545                }
4546        }, {
4547                .alg = "ctr(cast6)",
4548                .test = alg_test_skcipher,
4549                .suite = {
4550                        .cipher = __VECS(cast6_ctr_tv_template)
4551                }
4552        }, {
4553                .alg = "ctr(des)",
4554                .test = alg_test_skcipher,
4555                .suite = {
4556                        .cipher = __VECS(des_ctr_tv_template)
4557                }
4558        }, {
4559                .alg = "ctr(des3_ede)",
4560                .test = alg_test_skcipher,
4561                .fips_allowed = 1,
4562                .suite = {
4563                        .cipher = __VECS(des3_ede_ctr_tv_template)
4564                }
4565        }, {
4566                /* Same as ctr(aes) except the key is stored in
4567                 * hardware secure memory which we reference by index
4568                 */
4569                .alg = "ctr(paes)",
4570                .test = alg_test_null,
4571                .fips_allowed = 1,
4572        }, {
4573
4574                /* Same as ctr(sm4) except the key is stored in
4575                 * hardware secure memory which we reference by index
4576                 */
4577                .alg = "ctr(psm4)",
4578                .test = alg_test_null,
4579        }, {
4580                .alg = "ctr(serpent)",
4581                .test = alg_test_skcipher,
4582                .suite = {
4583                        .cipher = __VECS(serpent_ctr_tv_template)
4584                }
4585        }, {
4586                .alg = "ctr(sm4)",
4587                .test = alg_test_skcipher,
4588                .suite = {
4589                        .cipher = __VECS(sm4_ctr_tv_template)
4590                }
4591        }, {
4592                .alg = "ctr(twofish)",
4593                .test = alg_test_skcipher,
4594                .suite = {
4595                        .cipher = __VECS(tf_ctr_tv_template)
4596                }
4597        }, {
4598#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4599                .alg = "ctr-paes-s390",
4600                .fips_allowed = 1,
4601                .test = alg_test_skcipher,
4602                .suite = {
4603                        .cipher = __VECS(aes_ctr_tv_template)
4604                }
4605        }, {
4606#endif
4607                .alg = "cts(cbc(aes))",
4608                .test = alg_test_skcipher,
4609                .fips_allowed = 1,
4610                .suite = {
4611                        .cipher = __VECS(cts_mode_tv_template)
4612                }
4613        }, {
4614                /* Same as cts(cbc((aes)) except the key is stored in
4615                 * hardware secure memory which we reference by index
4616                 */
4617                .alg = "cts(cbc(paes))",
4618                .test = alg_test_null,
4619                .fips_allowed = 1,
4620        }, {
4621                .alg = "curve25519",
4622                .test = alg_test_kpp,
4623                .suite = {
4624                        .kpp = __VECS(curve25519_tv_template)
4625                }
4626        }, {
4627                .alg = "deflate",
4628                .test = alg_test_comp,
4629                .fips_allowed = 1,
4630                .suite = {
4631                        .comp = {
4632                                .comp = __VECS(deflate_comp_tv_template),
4633                                .decomp = __VECS(deflate_decomp_tv_template)
4634                        }
4635                }
4636        }, {
4637                .alg = "dh",
4638                .test = alg_test_kpp,
4639                .fips_allowed = 1,
4640                .suite = {
4641                        .kpp = __VECS(dh_tv_template)
4642                }
4643        }, {
4644                .alg = "digest_null",
4645                .test = alg_test_null,
4646        }, {
4647                .alg = "drbg_nopr_ctr_aes128",
4648                .test = alg_test_drbg,
4649                .fips_allowed = 1,
4650                .suite = {
4651                        .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
4652                }
4653        }, {
4654                .alg = "drbg_nopr_ctr_aes192",
4655                .test = alg_test_drbg,
4656                .fips_allowed = 1,
4657                .suite = {
4658                        .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
4659                }
4660        }, {
4661                .alg = "drbg_nopr_ctr_aes256",
4662                .test = alg_test_drbg,
4663                .fips_allowed = 1,
4664                .suite = {
4665                        .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
4666                }
4667        }, {
4668                /*
4669                 * There is no need to specifically test the DRBG with every
4670                 * backend cipher -- covered by drbg_nopr_hmac_sha256 test
4671                 */
4672                .alg = "drbg_nopr_hmac_sha1",
4673                .fips_allowed = 1,
4674                .test = alg_test_null,
4675        }, {
4676                .alg = "drbg_nopr_hmac_sha256",
4677                .test = alg_test_drbg,
4678                .fips_allowed = 1,
4679                .suite = {
4680                        .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
4681                }
4682        }, {
4683                /* covered by drbg_nopr_hmac_sha256 test */
4684                .alg = "drbg_nopr_hmac_sha384",
4685                .fips_allowed = 1,
4686                .test = alg_test_null,
4687        }, {
4688                .alg = "drbg_nopr_hmac_sha512",
4689                .test = alg_test_drbg,
4690                .fips_allowed = 1,
4691                .suite = {
4692                        .drbg = __VECS(drbg_nopr_hmac_sha512_tv_template)
4693                }
4694        }, {
4695                .alg = "drbg_nopr_sha1",
4696                .fips_allowed = 1,
4697                .test = alg_test_null,
4698        }, {
4699                .alg = "drbg_nopr_sha256",
4700                .test = alg_test_drbg,
4701                .fips_allowed = 1,
4702                .suite = {
4703                        .drbg = __VECS(drbg_nopr_sha256_tv_template)
4704                }
4705        }, {
4706                /* covered by drbg_nopr_sha256 test */
4707                .alg = "drbg_nopr_sha384",
4708                .fips_allowed = 1,
4709                .test = alg_test_null,
4710        }, {
4711                .alg = "drbg_nopr_sha512",
4712                .fips_allowed = 1,
4713                .test = alg_test_null,
4714        }, {
4715                .alg = "drbg_pr_ctr_aes128",
4716                .test = alg_test_drbg,
4717                .fips_allowed = 1,
4718                .suite = {
4719                        .drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
4720                }
4721        }, {
4722                /* covered by drbg_pr_ctr_aes128 test */
4723                .alg = "drbg_pr_ctr_aes192",
4724                .fips_allowed = 1,
4725                .test = alg_test_null,
4726        }, {
4727                .alg = "drbg_pr_ctr_aes256",
4728                .fips_allowed = 1,
4729                .test = alg_test_null,
4730        }, {
4731                .alg = "drbg_pr_hmac_sha1",
4732                .fips_allowed = 1,
4733                .test = alg_test_null,
4734        }, {
4735                .alg = "drbg_pr_hmac_sha256",
4736                .test = alg_test_drbg,
4737                .fips_allowed = 1,
4738                .suite = {
4739                        .drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
4740                }
4741        }, {
4742                /* covered by drbg_pr_hmac_sha256 test */
4743                .alg = "drbg_pr_hmac_sha384",
4744                .fips_allowed = 1,
4745                .test = alg_test_null,
4746        }, {
4747                .alg = "drbg_pr_hmac_sha512",
4748                .test = alg_test_null,
4749                .fips_allowed = 1,
4750        }, {
4751                .alg = "drbg_pr_sha1",
4752                .fips_allowed = 1,
4753                .test = alg_test_null,
4754        }, {
4755                .alg = "drbg_pr_sha256",
4756                .test = alg_test_drbg,
4757                .fips_allowed = 1,
4758                .suite = {
4759                        .drbg = __VECS(drbg_pr_sha256_tv_template)
4760                }
4761        }, {
4762                /* covered by drbg_pr_sha256 test */
4763                .alg = "drbg_pr_sha384",
4764                .fips_allowed = 1,
4765                .test = alg_test_null,
4766        }, {
4767                .alg = "drbg_pr_sha512",
4768                .fips_allowed = 1,
4769                .test = alg_test_null,
4770        }, {
4771                .alg = "ecb(aes)",
4772                .test = alg_test_skcipher,
4773                .fips_allowed = 1,
4774                .suite = {
4775                        .cipher = __VECS(aes_tv_template)
4776                }
4777        }, {
4778                .alg = "ecb(anubis)",
4779                .test = alg_test_skcipher,
4780                .suite = {
4781                        .cipher = __VECS(anubis_tv_template)
4782                }
4783        }, {
4784                .alg = "ecb(arc4)",
4785                .generic_driver = "ecb(arc4)-generic",
4786                .test = alg_test_skcipher,
4787                .suite = {
4788                        .cipher = __VECS(arc4_tv_template)
4789                }
4790        }, {
4791                .alg = "ecb(blowfish)",
4792                .test = alg_test_skcipher,
4793                .suite = {
4794                        .cipher = __VECS(bf_tv_template)
4795                }
4796        }, {
4797                .alg = "ecb(camellia)",
4798                .test = alg_test_skcipher,
4799                .suite = {
4800                        .cipher = __VECS(camellia_tv_template)
4801                }
4802        }, {
4803                .alg = "ecb(cast5)",
4804                .test = alg_test_skcipher,
4805                .suite = {
4806                        .cipher = __VECS(cast5_tv_template)
4807                }
4808        }, {
4809                .alg = "ecb(cast6)",
4810                .test = alg_test_skcipher,
4811                .suite = {
4812                        .cipher = __VECS(cast6_tv_template)
4813                }
4814        }, {
4815                .alg = "ecb(cipher_null)",
4816                .test = alg_test_null,
4817                .fips_allowed = 1,
4818        }, {
4819                .alg = "ecb(des)",
4820                .test = alg_test_skcipher,
4821                .suite = {
4822                        .cipher = __VECS(des_tv_template)
4823                }
4824        }, {
4825                .alg = "ecb(des3_ede)",
4826                .test = alg_test_skcipher,
4827                .fips_allowed = 1,
4828                .suite = {
4829                        .cipher = __VECS(des3_ede_tv_template)
4830                }
4831        }, {
4832                .alg = "ecb(fcrypt)",
4833                .test = alg_test_skcipher,
4834                .suite = {
4835                        .cipher = {
4836                                .vecs = fcrypt_pcbc_tv_template,
4837                                .count = 1
4838                        }
4839                }
4840        }, {
4841                .alg = "ecb(khazad)",
4842                .test = alg_test_skcipher,
4843                .suite = {
4844                        .cipher = __VECS(khazad_tv_template)
4845                }
4846        }, {
4847                /* Same as ecb(aes) except the key is stored in
4848                 * hardware secure memory which we reference by index
4849                 */
4850                .alg = "ecb(paes)",
4851                .test = alg_test_null,
4852                .fips_allowed = 1,
4853        }, {
4854                .alg = "ecb(seed)",
4855                .test = alg_test_skcipher,
4856                .suite = {
4857                        .cipher = __VECS(seed_tv_template)
4858                }
4859        }, {
4860                .alg = "ecb(serpent)",
4861                .test = alg_test_skcipher,
4862                .suite = {
4863                        .cipher = __VECS(serpent_tv_template)
4864                }
4865        }, {
4866                .alg = "ecb(sm4)",
4867                .test = alg_test_skcipher,
4868                .suite = {
4869                        .cipher = __VECS(sm4_tv_template)
4870                }
4871        }, {
4872                .alg = "ecb(tea)",
4873                .test = alg_test_skcipher,
4874                .suite = {
4875                        .cipher = __VECS(tea_tv_template)
4876                }
4877        }, {
4878                .alg = "ecb(twofish)",
4879                .test = alg_test_skcipher,
4880                .suite = {
4881                        .cipher = __VECS(tf_tv_template)
4882                }
4883        }, {
4884                .alg = "ecb(xeta)",
4885                .test = alg_test_skcipher,
4886                .suite = {
4887                        .cipher = __VECS(xeta_tv_template)
4888                }
4889        }, {
4890                .alg = "ecb(xtea)",
4891                .test = alg_test_skcipher,
4892                .suite = {
4893                        .cipher = __VECS(xtea_tv_template)
4894                }
4895        }, {
4896#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
4897                .alg = "ecb-paes-s390",
4898                .fips_allowed = 1,
4899                .test = alg_test_skcipher,
4900                .suite = {
4901                        .cipher = __VECS(aes_tv_template)
4902                }
4903        }, {
4904#endif
4905                .alg = "ecdh-nist-p192",
4906                .test = alg_test_kpp,
4907                .suite = {
4908                        .kpp = __VECS(ecdh_p192_tv_template)
4909                }
4910        }, {
4911                .alg = "ecdh-nist-p256",
4912                .test = alg_test_kpp,
4913                .fips_allowed = 1,
4914                .suite = {
4915                        .kpp = __VECS(ecdh_p256_tv_template)
4916                }
4917        }, {
4918                .alg = "ecdh-nist-p384",
4919                .test = alg_test_kpp,
4920                .fips_allowed = 1,
4921                .suite = {
4922                        .kpp = __VECS(ecdh_p384_tv_template)
4923                }
4924        }, {
4925                .alg = "ecdsa-nist-p192",
4926                .test = alg_test_akcipher,
4927                .suite = {
4928                        .akcipher = __VECS(ecdsa_nist_p192_tv_template)
4929                }
4930        }, {
4931                .alg = "ecdsa-nist-p256",
4932                .test = alg_test_akcipher,
4933                .suite = {
4934                        .akcipher = __VECS(ecdsa_nist_p256_tv_template)
4935                }
4936        }, {
4937                .alg = "ecdsa-nist-p384",
4938                .test = alg_test_akcipher,
4939                .suite = {
4940                        .akcipher = __VECS(ecdsa_nist_p384_tv_template)
4941                }
4942        }, {
4943                .alg = "ecrdsa",
4944                .test = alg_test_akcipher,
4945                .suite = {
4946                        .akcipher = __VECS(ecrdsa_tv_template)
4947                }
4948        }, {
4949                .alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",
4950                .test = alg_test_aead,
4951                .fips_allowed = 1,
4952                .suite = {
4953                        .aead = __VECS(essiv_hmac_sha256_aes_cbc_tv_temp)
4954                }
4955        }, {
4956                .alg = "essiv(cbc(aes),sha256)",
4957                .test = alg_test_skcipher,
4958                .fips_allowed = 1,
4959                .suite = {
4960                        .cipher = __VECS(essiv_aes_cbc_tv_template)
4961                }
4962        }, {
4963                .alg = "gcm(aes)",
4964                .generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",
4965                .test = alg_test_aead,
4966                .fips_allowed = 1,
4967                .suite = {
4968                        .aead = __VECS(aes_gcm_tv_template)
4969                }
4970        }, {
4971                .alg = "ghash",
4972                .test = alg_test_hash,
4973                .fips_allowed = 1,
4974                .suite = {
4975                        .hash = __VECS(ghash_tv_template)
4976                }
4977        }, {
4978                .alg = "hmac(md5)",
4979                .test = alg_test_hash,
4980                .suite = {
4981                        .hash = __VECS(hmac_md5_tv_template)
4982                }
4983        }, {
4984                .alg = "hmac(rmd160)",
4985                .test = alg_test_hash,
4986                .suite = {
4987                        .hash = __VECS(hmac_rmd160_tv_template)
4988                }
4989        }, {
4990                .alg = "hmac(sha1)",
4991                .test = alg_test_hash,
4992                .fips_allowed = 1,
4993                .suite = {
4994                        .hash = __VECS(hmac_sha1_tv_template)
4995                }
4996        }, {
4997                .alg = "hmac(sha224)",
4998                .test = alg_test_hash,
4999                .fips_allowed = 1,
5000                .suite = {
5001                        .hash = __VECS(hmac_sha224_tv_template)
5002                }
5003        }, {
5004                .alg = "hmac(sha256)",
5005                .test = alg_test_hash,
5006                .fips_allowed = 1,
5007                .suite = {
5008                        .hash = __VECS(hmac_sha256_tv_template)
5009                }
5010        }, {
5011                .alg = "hmac(sha3-224)",
5012                .test = alg_test_hash,
5013                .fips_allowed = 1,
5014                .suite = {
5015                        .hash = __VECS(hmac_sha3_224_tv_template)
5016                }
5017        }, {
5018                .alg = "hmac(sha3-256)",
5019                .test = alg_test_hash,
5020                .fips_allowed = 1,
5021                .suite = {
5022                        .hash = __VECS(hmac_sha3_256_tv_template)
5023                }
5024        }, {
5025                .alg = "hmac(sha3-384)",
5026                .test = alg_test_hash,
5027                .fips_allowed = 1,
5028                .suite = {
5029                        .hash = __VECS(hmac_sha3_384_tv_template)
5030                }
5031        }, {
5032                .alg = "hmac(sha3-512)",
5033                .test = alg_test_hash,
5034                .fips_allowed = 1,
5035                .suite = {
5036                        .hash = __VECS(hmac_sha3_512_tv_template)
5037                }
5038        }, {
5039                .alg = "hmac(sha384)",
5040                .test = alg_test_hash,
5041                .fips_allowed = 1,
5042                .suite = {
5043                        .hash = __VECS(hmac_sha384_tv_template)
5044                }
5045        }, {
5046                .alg = "hmac(sha512)",
5047                .test = alg_test_hash,
5048                .fips_allowed = 1,
5049                .suite = {
5050                        .hash = __VECS(hmac_sha512_tv_template)
5051                }
5052        }, {
5053                .alg = "hmac(sm3)",
5054                .test = alg_test_hash,
5055                .suite = {
5056                        .hash = __VECS(hmac_sm3_tv_template)
5057                }
5058        }, {
5059                .alg = "hmac(streebog256)",
5060                .test = alg_test_hash,
5061                .suite = {
5062                        .hash = __VECS(hmac_streebog256_tv_template)
5063                }
5064        }, {
5065                .alg = "hmac(streebog512)",
5066                .test = alg_test_hash,
5067                .suite = {
5068                        .hash = __VECS(hmac_streebog512_tv_template)
5069                }
5070        }, {
5071                .alg = "jitterentropy_rng",
5072                .fips_allowed = 1,
5073                .test = alg_test_null,
5074        }, {
5075                .alg = "kw(aes)",
5076                .test = alg_test_skcipher,
5077                .fips_allowed = 1,
5078                .suite = {
5079                        .cipher = __VECS(aes_kw_tv_template)
5080                }
5081        }, {
5082                .alg = "lrw(aes)",
5083                .generic_driver = "lrw(ecb(aes-generic))",
5084                .test = alg_test_skcipher,
5085                .suite = {
5086                        .cipher = __VECS(aes_lrw_tv_template)
5087                }
5088        }, {
5089                .alg = "lrw(camellia)",
5090                .generic_driver = "lrw(ecb(camellia-generic))",
5091                .test = alg_test_skcipher,
5092                .suite = {
5093                        .cipher = __VECS(camellia_lrw_tv_template)
5094                }
5095        }, {
5096                .alg = "lrw(cast6)",
5097                .generic_driver = "lrw(ecb(cast6-generic))",
5098                .test = alg_test_skcipher,
5099                .suite = {
5100                        .cipher = __VECS(cast6_lrw_tv_template)
5101                }
5102        }, {
5103                .alg = "lrw(serpent)",
5104                .generic_driver = "lrw(ecb(serpent-generic))",
5105                .test = alg_test_skcipher,
5106                .suite = {
5107                        .cipher = __VECS(serpent_lrw_tv_template)
5108                }
5109        }, {
5110                .alg = "lrw(twofish)",
5111                .generic_driver = "lrw(ecb(twofish-generic))",
5112                .test = alg_test_skcipher,
5113                .suite = {
5114                        .cipher = __VECS(tf_lrw_tv_template)
5115                }
5116        }, {
5117                .alg = "lz4",
5118                .test = alg_test_comp,
5119                .fips_allowed = 1,
5120                .suite = {
5121                        .comp = {
5122                                .comp = __VECS(lz4_comp_tv_template),
5123                                .decomp = __VECS(lz4_decomp_tv_template)
5124                        }
5125                }
5126        }, {
5127                .alg = "lz4hc",
5128                .test = alg_test_comp,
5129                .fips_allowed = 1,
5130                .suite = {
5131                        .comp = {
5132                                .comp = __VECS(lz4hc_comp_tv_template),
5133                                .decomp = __VECS(lz4hc_decomp_tv_template)
5134                        }
5135                }
5136        }, {
5137                .alg = "lzo",
5138                .test = alg_test_comp,
5139                .fips_allowed = 1,
5140                .suite = {
5141                        .comp = {
5142                                .comp = __VECS(lzo_comp_tv_template),
5143                                .decomp = __VECS(lzo_decomp_tv_template)
5144                        }
5145                }
5146        }, {
5147                .alg = "lzo-rle",
5148                .test = alg_test_comp,
5149                .fips_allowed = 1,
5150                .suite = {
5151                        .comp = {
5152                                .comp = __VECS(lzorle_comp_tv_template),
5153                                .decomp = __VECS(lzorle_decomp_tv_template)
5154                        }
5155                }
5156        }, {
5157                .alg = "md4",
5158                .test = alg_test_hash,
5159                .suite = {
5160                        .hash = __VECS(md4_tv_template)
5161                }
5162        }, {
5163                .alg = "md5",
5164                .test = alg_test_hash,
5165                .suite = {
5166                        .hash = __VECS(md5_tv_template)
5167                }
5168        }, {
5169                .alg = "michael_mic",
5170                .test = alg_test_hash,
5171                .suite = {
5172                        .hash = __VECS(michael_mic_tv_template)
5173                }
5174        }, {
5175                .alg = "nhpoly1305",
5176                .test = alg_test_hash,
5177                .suite = {
5178                        .hash = __VECS(nhpoly1305_tv_template)
5179                }
5180        }, {
5181                .alg = "ofb(aes)",
5182                .test = alg_test_skcipher,
5183                .fips_allowed = 1,
5184                .suite = {
5185                        .cipher = __VECS(aes_ofb_tv_template)
5186                }
5187        }, {
5188                /* Same as ofb(aes) except the key is stored in
5189                 * hardware secure memory which we reference by index
5190                 */
5191                .alg = "ofb(paes)",
5192                .test = alg_test_null,
5193                .fips_allowed = 1,
5194        }, {
5195                .alg = "ofb(sm4)",
5196                .test = alg_test_skcipher,
5197                .suite = {
5198                        .cipher = __VECS(sm4_ofb_tv_template)
5199                }
5200        }, {
5201                .alg = "pcbc(fcrypt)",
5202                .test = alg_test_skcipher,
5203                .suite = {
5204                        .cipher = __VECS(fcrypt_pcbc_tv_template)
5205                }
5206        }, {
5207                .alg = "pkcs1pad(rsa,sha224)",
5208                .test = alg_test_null,
5209                .fips_allowed = 1,
5210        }, {
5211                .alg = "pkcs1pad(rsa,sha256)",
5212                .test = alg_test_akcipher,
5213                .fips_allowed = 1,
5214                .suite = {
5215                        .akcipher = __VECS(pkcs1pad_rsa_tv_template)
5216                }
5217        }, {
5218                .alg = "pkcs1pad(rsa,sha384)",
5219                .test = alg_test_null,
5220                .fips_allowed = 1,
5221        }, {
5222                .alg = "pkcs1pad(rsa,sha512)",
5223                .test = alg_test_null,
5224                .fips_allowed = 1,
5225        }, {
5226                .alg = "poly1305",
5227                .test = alg_test_hash,
5228                .suite = {
5229                        .hash = __VECS(poly1305_tv_template)
5230                }
5231        }, {
5232                .alg = "rfc3686(ctr(aes))",
5233                .test = alg_test_skcipher,
5234                .fips_allowed = 1,
5235                .suite = {
5236                        .cipher = __VECS(aes_ctr_rfc3686_tv_template)
5237                }
5238        }, {
5239                .alg = "rfc3686(ctr(sm4))",
5240                .test = alg_test_skcipher,
5241                .suite = {
5242                        .cipher = __VECS(sm4_ctr_rfc3686_tv_template)
5243                }
5244        }, {
5245                .alg = "rfc4106(gcm(aes))",
5246                .generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",
5247                .test = alg_test_aead,
5248                .fips_allowed = 1,
5249                .suite = {
5250                        .aead = {
5251                                ____VECS(aes_gcm_rfc4106_tv_template),
5252                                .einval_allowed = 1,
5253                                .aad_iv = 1,
5254                        }
5255                }
5256        }, {
5257                .alg = "rfc4309(ccm(aes))",
5258                .generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",
5259                .test = alg_test_aead,
5260                .fips_allowed = 1,
5261                .suite = {
5262                        .aead = {
5263                                ____VECS(aes_ccm_rfc4309_tv_template),
5264                                .einval_allowed = 1,
5265                                .aad_iv = 1,
5266                        }
5267                }
5268        }, {
5269                .alg = "rfc4543(gcm(aes))",
5270                .generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",
5271                .test = alg_test_aead,
5272                .suite = {
5273                        .aead = {
5274                                ____VECS(aes_gcm_rfc4543_tv_template),
5275                                .einval_allowed = 1,
5276                                .aad_iv = 1,
5277                        }
5278                }
5279        }, {
5280                .alg = "rfc7539(chacha20,poly1305)",
5281                .test = alg_test_aead,
5282                .suite = {
5283                        .aead = __VECS(rfc7539_tv_template)
5284                }
5285        }, {
5286                .alg = "rfc7539esp(chacha20,poly1305)",
5287                .test = alg_test_aead,
5288                .suite = {
5289                        .aead = {
5290                                ____VECS(rfc7539esp_tv_template),
5291                                .einval_allowed = 1,
5292                                .aad_iv = 1,
5293                        }
5294                }
5295        }, {
5296                .alg = "rmd160",
5297                .test = alg_test_hash,
5298                .suite = {
5299                        .hash = __VECS(rmd160_tv_template)
5300                }
5301        }, {
5302                .alg = "rsa",
5303                .test = alg_test_akcipher,
5304                .fips_allowed = 1,
5305                .suite = {
5306                        .akcipher = __VECS(rsa_tv_template)
5307                }
5308        }, {
5309                .alg = "sha1",
5310                .test = alg_test_hash,
5311                .fips_allowed = 1,
5312                .suite = {
5313                        .hash = __VECS(sha1_tv_template)
5314                }
5315        }, {
5316                .alg = "sha224",
5317                .test = alg_test_hash,
5318                .fips_allowed = 1,
5319                .suite = {
5320                        .hash = __VECS(sha224_tv_template)
5321                }
5322        }, {
5323                .alg = "sha256",
5324                .test = alg_test_hash,
5325                .fips_allowed = 1,
5326                .suite = {
5327                        .hash = __VECS(sha256_tv_template)
5328                }
5329        }, {
5330                .alg = "sha3-224",
5331                .test = alg_test_hash,
5332                .fips_allowed = 1,
5333                .suite = {
5334                        .hash = __VECS(sha3_224_tv_template)
5335                }
5336        }, {
5337                .alg = "sha3-256",
5338                .test = alg_test_hash,
5339                .fips_allowed = 1,
5340                .suite = {
5341                        .hash = __VECS(sha3_256_tv_template)
5342                }
5343        }, {
5344                .alg = "sha3-384",
5345                .test = alg_test_hash,
5346                .fips_allowed = 1,
5347                .suite = {
5348                        .hash = __VECS(sha3_384_tv_template)
5349                }
5350        }, {
5351                .alg = "sha3-512",
5352                .test = alg_test_hash,
5353                .fips_allowed = 1,
5354                .suite = {
5355                        .hash = __VECS(sha3_512_tv_template)
5356                }
5357        }, {
5358                .alg = "sha384",
5359                .test = alg_test_hash,
5360                .fips_allowed = 1,
5361                .suite = {
5362                        .hash = __VECS(sha384_tv_template)
5363                }
5364        }, {
5365                .alg = "sha512",
5366                .test = alg_test_hash,
5367                .fips_allowed = 1,
5368                .suite = {
5369                        .hash = __VECS(sha512_tv_template)
5370                }
5371        }, {
5372                .alg = "sm2",
5373                .test = alg_test_akcipher,
5374                .suite = {
5375                        .akcipher = __VECS(sm2_tv_template)
5376                }
5377        }, {
5378                .alg = "sm3",
5379                .test = alg_test_hash,
5380                .suite = {
5381                        .hash = __VECS(sm3_tv_template)
5382                }
5383        }, {
5384                .alg = "streebog256",
5385                .test = alg_test_hash,
5386                .suite = {
5387                        .hash = __VECS(streebog256_tv_template)
5388                }
5389        }, {
5390                .alg = "streebog512",
5391                .test = alg_test_hash,
5392                .suite = {
5393                        .hash = __VECS(streebog512_tv_template)
5394                }
5395        }, {
5396                .alg = "vmac64(aes)",
5397                .test = alg_test_hash,
5398                .suite = {
5399                        .hash = __VECS(vmac64_aes_tv_template)
5400                }
5401        }, {
5402                .alg = "wp256",
5403                .test = alg_test_hash,
5404                .suite = {
5405                        .hash = __VECS(wp256_tv_template)
5406                }
5407        }, {
5408                .alg = "wp384",
5409                .test = alg_test_hash,
5410                .suite = {
5411                        .hash = __VECS(wp384_tv_template)
5412                }
5413        }, {
5414                .alg = "wp512",
5415                .test = alg_test_hash,
5416                .suite = {
5417                        .hash = __VECS(wp512_tv_template)
5418                }
5419        }, {
5420                .alg = "xcbc(aes)",
5421                .test = alg_test_hash,
5422                .suite = {
5423                        .hash = __VECS(aes_xcbc128_tv_template)
5424                }
5425        }, {
5426                .alg = "xchacha12",
5427                .test = alg_test_skcipher,
5428                .suite = {
5429                        .cipher = __VECS(xchacha12_tv_template)
5430                },
5431        }, {
5432                .alg = "xchacha20",
5433                .test = alg_test_skcipher,
5434                .suite = {
5435                        .cipher = __VECS(xchacha20_tv_template)
5436                },
5437        }, {
5438                .alg = "xts(aes)",
5439                .generic_driver = "xts(ecb(aes-generic))",
5440                .test = alg_test_skcipher,
5441                .fips_allowed = 1,
5442                .suite = {
5443                        .cipher = __VECS(aes_xts_tv_template)
5444                }
5445        }, {
5446                .alg = "xts(camellia)",
5447                .generic_driver = "xts(ecb(camellia-generic))",
5448                .test = alg_test_skcipher,
5449                .suite = {
5450                        .cipher = __VECS(camellia_xts_tv_template)
5451                }
5452        }, {
5453                .alg = "xts(cast6)",
5454                .generic_driver = "xts(ecb(cast6-generic))",
5455                .test = alg_test_skcipher,
5456                .suite = {
5457                        .cipher = __VECS(cast6_xts_tv_template)
5458                }
5459        }, {
5460                /* Same as xts(aes) except the key is stored in
5461                 * hardware secure memory which we reference by index
5462                 */
5463                .alg = "xts(paes)",
5464                .test = alg_test_null,
5465                .fips_allowed = 1,
5466        }, {
5467                .alg = "xts(serpent)",
5468                .generic_driver = "xts(ecb(serpent-generic))",
5469                .test = alg_test_skcipher,
5470                .suite = {
5471                        .cipher = __VECS(serpent_xts_tv_template)
5472                }
5473        }, {
5474                .alg = "xts(twofish)",
5475                .generic_driver = "xts(ecb(twofish-generic))",
5476                .test = alg_test_skcipher,
5477                .suite = {
5478                        .cipher = __VECS(tf_xts_tv_template)
5479                }
5480        }, {
5481#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)
5482                .alg = "xts-paes-s390",
5483                .fips_allowed = 1,
5484                .test = alg_test_skcipher,
5485                .suite = {
5486                        .cipher = __VECS(aes_xts_tv_template)
5487                }
5488        }, {
5489#endif
5490                .alg = "xts4096(paes)",
5491                .test = alg_test_null,
5492                .fips_allowed = 1,
5493        }, {
5494                .alg = "xts512(paes)",
5495                .test = alg_test_null,
5496                .fips_allowed = 1,
5497        }, {
5498                .alg = "xxhash64",
5499                .test = alg_test_hash,
5500                .fips_allowed = 1,
5501                .suite = {
5502                        .hash = __VECS(xxhash64_tv_template)
5503                }
5504        }, {
5505                .alg = "zlib-deflate",
5506                .test = alg_test_comp,
5507                .fips_allowed = 1,
5508                .suite = {
5509                        .comp = {
5510                                .comp = __VECS(zlib_deflate_comp_tv_template),
5511                                .decomp = __VECS(zlib_deflate_decomp_tv_template)
5512                        }
5513                }
5514        }, {
5515                .alg = "zstd",
5516                .test = alg_test_comp,
5517                .fips_allowed = 1,
5518                .suite = {
5519                        .comp = {
5520                                .comp = __VECS(zstd_comp_tv_template),
5521                                .decomp = __VECS(zstd_decomp_tv_template)
5522                        }
5523                }
5524        }
5525};
5526
5527static void alg_check_test_descs_order(void)
5528{
5529        int i;
5530
5531        for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {
5532                int diff = strcmp(alg_test_descs[i - 1].alg,
5533                                  alg_test_descs[i].alg);
5534
5535                if (WARN_ON(diff > 0)) {
5536                        pr_warn("testmgr: alg_test_descs entries in wrong order: '%s' before '%s'\n",
5537                                alg_test_descs[i - 1].alg,
5538                                alg_test_descs[i].alg);
5539                }
5540
5541                if (WARN_ON(diff == 0)) {
5542                        pr_warn("testmgr: duplicate alg_test_descs entry: '%s'\n",
5543                                alg_test_descs[i].alg);
5544                }
5545        }
5546}
5547
5548static void alg_check_testvec_configs(void)
5549{
5550        int i;
5551
5552        for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)
5553                WARN_ON(!valid_testvec_config(
5554                                &default_cipher_testvec_configs[i]));
5555
5556        for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)
5557                WARN_ON(!valid_testvec_config(
5558                                &default_hash_testvec_configs[i]));
5559}
5560
5561static void testmgr_onetime_init(void)
5562{
5563        alg_check_test_descs_order();
5564        alg_check_testvec_configs();
5565
5566#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS
5567        pr_warn("alg: extra crypto tests enabled.  This is intended for developer use only.\n");
5568#endif
5569}
5570
5571static int alg_find_test(const char *alg)
5572{
5573        int start = 0;
5574        int end = ARRAY_SIZE(alg_test_descs);
5575
5576        while (start < end) {
5577                int i = (start + end) / 2;
5578                int diff = strcmp(alg_test_descs[i].alg, alg);
5579
5580                if (diff > 0