// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2016 Broadcom
 */

#include <linux/kernel.h>
#include <linux/string.h>

#include "util.h"
#include "spu.h"
#include "spum.h"
#include "cipher.h"

char *hash_alg_name[] = { "None", "md5", "sha1", "sha224", "sha256", "aes",
        "sha384", "sha512", "sha3_224", "sha3_256", "sha3_384", "sha3_512" };

char *aead_alg_name[] = { "ccm(aes)", "gcm(aes)", "authenc" };

/* Assumes SPU-M messages are in big endian */
void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len)
{
        u8 *ptr = buf;
        struct SPUHEADER *spuh = (struct SPUHEADER *)buf;
        unsigned int hash_key_len = 0;
        unsigned int hash_state_len = 0;
        unsigned int cipher_key_len = 0;
        unsigned int iv_len;
        u32 pflags;
        u32 cflags;
        u32 ecf;
        u32 cipher_alg;
        u32 cipher_mode;
        u32 cipher_type;
        u32 hash_alg;
        u32 hash_mode;
        u32 hash_type;
        u32 sctx_size;   /* SCTX length in words */
        u32 sctx_pl_len; /* SCTX payload length in bytes */

        packet_log("\n");
        packet_log("SPU Message header %p len: %u\n", buf, buf_len);

        /* ========== Decode MH ========== */
        packet_log("  MH 0x%08x\n", be32_to_cpup((__be32 *)ptr));
        if (spuh->mh.flags & MH_SCTX_PRES)
                packet_log("    SCTX  present\n");
        if (spuh->mh.flags & MH_BDESC_PRES)
                packet_log("    BDESC present\n");
        if (spuh->mh.flags & MH_MFM_PRES)
                packet_log("    MFM   present\n");
        if (spuh->mh.flags & MH_BD_PRES)
                packet_log("    BD    present\n");
        if (spuh->mh.flags & MH_HASH_PRES)
                packet_log("    HASH  present\n");
        if (spuh->mh.flags & MH_SUPDT_PRES)
                packet_log("    SUPDT present\n");
        packet_log("    Opcode 0x%02x\n", spuh->mh.op_code);

        ptr += sizeof(spuh->mh) + sizeof(spuh->emh);  /* skip emh. unused */

        /* ========== Decode SCTX ========== */
        if (spuh->mh.flags & MH_SCTX_PRES) {
                pflags = be32_to_cpu(spuh->sa.proto_flags);
                packet_log("  SCTX[0] 0x%08x\n", pflags);
                sctx_size = pflags & SCTX_SIZE;
                packet_log("    Size %u words\n", sctx_size);

                cflags = be32_to_cpu(spuh->sa.cipher_flags);
                packet_log("  SCTX[1] 0x%08x\n", cflags);
                packet_log("    Inbound:%lu (1:decrypt/vrfy 0:encrypt/auth)\n",
                           (cflags & CIPHER_INBOUND) >> CIPHER_INBOUND_SHIFT);
                packet_log("    Order:%lu (1:AuthFirst 0:EncFirst)\n",
                           (cflags & CIPHER_ORDER) >> CIPHER_ORDER_SHIFT);
                packet_log("    ICV_IS_512:%lx\n",
                           (cflags & ICV_IS_512) >> ICV_IS_512_SHIFT);
                cipher_alg = (cflags & CIPHER_ALG) >> CIPHER_ALG_SHIFT;
                cipher_mode = (cflags & CIPHER_MODE) >> CIPHER_MODE_SHIFT;
                cipher_type = (cflags & CIPHER_TYPE) >> CIPHER_TYPE_SHIFT;
                packet_log("    Crypto Alg:%u Mode:%u Type:%u\n",
                           cipher_alg, cipher_mode, cipher_type);
                hash_alg = (cflags & HASH_ALG) >> HASH_ALG_SHIFT;
                hash_mode = (cflags & HASH_MODE) >> HASH_MODE_SHIFT;
                hash_type = (cflags & HASH_TYPE) >> HASH_TYPE_SHIFT;
                packet_log("    Hash   Alg:%x Mode:%x Type:%x\n",
                           hash_alg, hash_mode, hash_type);
                packet_log("    UPDT_Offset:%u\n", cflags & UPDT_OFST);

                ecf = be32_to_cpu(spuh->sa.ecf);
                packet_log("  SCTX[2] 0x%08x\n", ecf);
                packet_log("    WriteICV:%lu CheckICV:%lu ICV_SIZE:%u ",
                           (ecf & INSERT_ICV) >> INSERT_ICV_SHIFT,
                           (ecf & CHECK_ICV) >> CHECK_ICV_SHIFT,
                           (ecf & ICV_SIZE) >> ICV_SIZE_SHIFT);
                packet_log("BD_SUPPRESS:%lu\n",
                           (ecf & BD_SUPPRESS) >> BD_SUPPRESS_SHIFT);
                packet_log("    SCTX_IV:%lu ExplicitIV:%lu GenIV:%lu ",
                           (ecf & SCTX_IV) >> SCTX_IV_SHIFT,
                           (ecf & EXPLICIT_IV) >> EXPLICIT_IV_SHIFT,
                           (ecf & GEN_IV) >> GEN_IV_SHIFT);
                packet_log("IV_OV_OFST:%lu EXP_IV_SIZE:%u\n",
                           (ecf & IV_OFFSET) >> IV_OFFSET_SHIFT,
                           ecf & EXP_IV_SIZE);

                ptr += sizeof(struct SCTX);

                if (hash_alg && hash_mode) {
                        char *name = "NONE";

                        switch (hash_alg) {
                        case HASH_ALG_MD5:
                                hash_key_len = 16;
                                name = "MD5";
                                break;
                        case HASH_ALG_SHA1:
                                hash_key_len = 20;
                                name = "SHA1";
                                break;
                        case HASH_ALG_SHA224:
                                hash_key_len = 28;
                                name = "SHA224";
                                break;
                        case HASH_ALG_SHA256:
                                hash_key_len = 32;
                                name = "SHA256";
                                break;
                        case HASH_ALG_SHA384:
                                hash_key_len = 48;
                                name = "SHA384";
                                break;
                        case HASH_ALG_SHA512:
                                hash_key_len = 64;
                                name = "SHA512";
                                break;
                        case HASH_ALG_AES:
                                hash_key_len = 0;
                                name = "AES";
                                break;
                        case HASH_ALG_NONE:
                                break;
                        }

                        packet_log("    Auth Key Type:%s Length:%u Bytes\n",
                                   name, hash_key_len);
                        packet_dump("    KEY: ", ptr, hash_key_len);
                        ptr += hash_key_len;
                } else if ((hash_alg == HASH_ALG_AES) &&
                           (hash_mode == HASH_MODE_XCBC)) {
                        char *name = "NONE";

                        switch (cipher_type) {
                        case CIPHER_TYPE_AES128:
                                hash_key_len = 16;
                                name = "AES128-XCBC";
                                break;
                        case CIPHER_TYPE_AES192:
                                hash_key_len = 24;
                                name = "AES192-XCBC";
                                break;
                        case CIPHER_TYPE_AES256:
                                hash_key_len = 32;
                                name = "AES256-XCBC";
                                break;
                        }
                        packet_log("    Auth Key Type:%s Length:%u Bytes\n",
                                   name, hash_key_len);
                        packet_dump("    KEY: ", ptr, hash_key_len);
                        ptr += hash_key_len;
                }

                if (hash_alg && (hash_mode == HASH_MODE_NONE) &&
                    (hash_type == HASH_TYPE_UPDT)) {
                        char *name = "NONE";

                        switch (hash_alg) {
                        case HASH_ALG_MD5:
                                hash_state_len = 16;
                                name = "MD5";
                                break;
                        case HASH_ALG_SHA1:
                                hash_state_len = 20;
                                name = "SHA1";
                                break;
                        case HASH_ALG_SHA224:
                                hash_state_len = 32;
                                name = "SHA224";
                                break;
                        case HASH_ALG_SHA256:
                                hash_state_len = 32;
                                name = "SHA256";
                                break;
                        case HASH_ALG_SHA384:
                                hash_state_len = 48;
                                name = "SHA384";
                                break;
                        case HASH_ALG_SHA512:
                                hash_state_len = 64;
                                name = "SHA512";
                                break;
                        case HASH_ALG_AES:
                                hash_state_len = 0;
                                name = "AES";
                                break;
                        case HASH_ALG_NONE:
                                break;
                        }

                        packet_log("    Auth State Type:%s Length:%u Bytes\n",
                                   name, hash_state_len);
                        packet_dump("    State: ", ptr, hash_state_len);
                        ptr += hash_state_len;
                }

                if (cipher_alg) {
                        char *name = "NONE";

                        switch (cipher_alg) {
                        case CIPHER_ALG_DES:
                                cipher_key_len = 8;
                                name = "DES";
                                break;
                        case CIPHER_ALG_3DES:
                                cipher_key_len = 24;
                                name = "3DES";
                                break;
                        case CIPHER_ALG_AES:
                                switch (cipher_type) {
                                case CIPHER_TYPE_AES128:
                                        cipher_key_len = 16;
                                        name = "AES128";
                                        break;
                                case CIPHER_TYPE_AES192:
                                        cipher_key_len = 24;
                                        name = "AES192";
                                        break;
                                case CIPHER_TYPE_AES256:
                                        cipher_key_len = 32;
                                        name = "AES256";
                                        break;
                                }
                                break;
                        case CIPHER_ALG_NONE:
                                break;
                        }

                        packet_log("    Cipher Key Type:%s Length:%u Bytes\n",
                                   name, cipher_key_len);

                        /* XTS has two keys */
                        if (cipher_mode == CIPHER_MODE_XTS) {
                                packet_dump("    KEY2: ", ptr, cipher_key_len);
                                ptr += cipher_key_len;
                                packet_dump("    KEY1: ", ptr, cipher_key_len);
                                ptr += cipher_key_len;

                                cipher_key_len *= 2;
                        } else {
                                packet_dump("    KEY: ", ptr, cipher_key_len);
                                ptr += cipher_key_len;
                        }

                        if (ecf & SCTX_IV) {
                                sctx_pl_len = sctx_size * sizeof(u32) -
                                        sizeof(struct SCTX);
                                iv_len = sctx_pl_len -
                                        (hash_key_len + hash_state_len +
                                         cipher_key_len);
                                packet_log("    IV Length:%u Bytes\n", iv_len);
                                packet_dump("    IV: ", ptr, iv_len);
                                ptr += iv_len;
                        }
                }
        }

        /* ========== Decode BDESC ========== */
        if (spuh->mh.flags & MH_BDESC_PRES) {
                struct BDESC_HEADER *bdesc = (struct BDESC_HEADER *)ptr;

                packet_log("  BDESC[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr));
                packet_log("    OffsetMAC:%u LengthMAC:%u\n",
                           be16_to_cpu(bdesc->offset_mac),
                           be16_to_cpu(bdesc->length_mac));
                ptr += sizeof(u32);

                packet_log("  BDESC[1] 0x%08x\n", be32_to_cpup((__be32 *)ptr));
                packet_log("    OffsetCrypto:%u LengthCrypto:%u\n",
                           be16_to_cpu(bdesc->offset_crypto),
                           be16_to_cpu(bdesc->length_crypto));
                ptr += sizeof(u32);

                packet_log("  BDESC[2] 0x%08x\n", be32_to_cpup((__be32 *)ptr));
                packet_log("    OffsetICV:%u OffsetIV:%u\n",
                           be16_to_cpu(bdesc->offset_icv),
                           be16_to_cpu(bdesc->offset_iv));
                ptr += sizeof(u32);
        }

        /* ========== Decode BD ========== */
        if (spuh->mh.flags & MH_BD_PRES) {
                struct BD_HEADER *bd = (struct BD_HEADER *)ptr;

                packet_log("  BD[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr));
                packet_log("    Size:%ubytes PrevLength:%u\n",
                           be16_to_cpu(bd->size), be16_to_cpu(bd->prev_length));
                ptr += 4;
        }

        /* Double check sanity */
        if (buf + buf_len != ptr) {
                packet_log(" Packet parsed incorrectly. ");
                packet_log("buf:%p buf_len:%u buf+buf_len:%p ptr:%p\n",
                           buf, buf_len, buf + buf_len, ptr);
        }

        packet_log("\n");
}

/**
 * spum_ns2_ctx_max_payload() - Determine the max length of the payload for a
 * SPU message for a given cipher and hash alg context.
 * @cipher_alg:         The cipher algorithm
 * @cipher_mode:        The cipher mode
 * @blocksize:          The size of a block of data for this algo
 *
 * The max payload must be a multiple of the blocksize so that if a request is
 * too large to fit in a single SPU message, the request can be broken into
 * max_payload sized chunks. Each chunk must be a multiple of blocksize.
 *
 * Return: Max payload length in bytes
 */
u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
                             enum spu_cipher_mode cipher_mode,
                             unsigned int blocksize)
{
        u32 max_payload = SPUM_NS2_MAX_PAYLOAD;
        u32 excess;

        /* In XTS on SPU-M, we'll need to insert tweak before input data */
        if (cipher_mode == CIPHER_MODE_XTS)
                max_payload -= SPU_XTS_TWEAK_SIZE;

        excess = max_payload % blocksize;

        return max_payload - excess;
}

/**
 * spum_nsp_ctx_max_payload() - Determine the max length of the payload for a
 * SPU message for a given cipher and hash alg context.
 * @cipher_alg:         The cipher algorithm
 * @cipher_mode:        The cipher mode
 * @blocksize:          The size of a block of data for this algo
 *
 * The max payload must be a multiple of the blocksize so that if a request is
 * too large to fit in a single SPU message, the request can be broken into
 * max_payload sized chunks. Each chunk must be a multiple of blocksize.
 *
 * Return: Max payload length in bytes
 */
u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg,
                             enum spu_cipher_mode cipher_mode,
                             unsigned int blocksize)
{
        u32 max_payload = SPUM_NSP_MAX_PAYLOAD;
        u32 excess;

        /* In XTS on SPU-M, we'll need to insert tweak before input data */
        if (cipher_mode == CIPHER_MODE_XTS)
                max_payload -= SPU_XTS_TWEAK_SIZE;

        excess = max_payload % blocksize;

        return max_payload - excess;
}

/** spum_payload_length() - Given a SPU-M message header, extract the payload
 * length.
 * @spu_hdr:    Start of SPU header
 *
 * Assumes just MH, EMH, BD (no SCTX, BDESC. Works for response frames.
 *
 * Return: payload length in bytes
 */
u32 spum_payload_length(u8 *spu_hdr)
{
        struct BD_HEADER *bd;
        u32 pl_len;

        /* Find BD header.  skip MH, EMH */
        bd = (struct BD_HEADER *)(spu_hdr + 8);
        pl_len = be16_to_cpu(bd->size);

        return pl_len;
}

/**
 * spum_response_hdr_len() - Given the length of the hash key and encryption
 * key, determine the expected length of a SPU response header.
 * @auth_key_len:       authentication key length (bytes)
 * @enc_key_len:        encryption key length (bytes)
 * @is_hash:            true if response message is for a hash operation
 *
 * Return: length of SPU response header (bytes)
 */
u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash)
{
        if (is_hash)
                return SPU_HASH_RESP_HDR_LEN;
        else
                return SPU_RESP_HDR_LEN;
}

/**
 * spum_hash_pad_len() - Calculate the length of hash padding required to extend
 * data to a full block size.
 * @hash_alg:   hash algorithm
 * @hash_mode:       hash mode
 * @chunksize:  length of data, in bytes
 * @hash_block_size:  size of a block of data for hash algorithm
 *
 * Reserve space for 1 byte (0x80) start of pad and the total length as u64
 *
 * Return:  length of hash pad in bytes
 */
u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
                      u32 chunksize, u16 hash_block_size)
{
        unsigned int length_len;
        unsigned int used_space_last_block;
        int hash_pad_len;

        /* AES-XCBC hash requires just padding to next block boundary */
        if ((hash_alg == HASH_ALG_AES) && (hash_mode == HASH_MODE_XCBC)) {
                used_space_last_block = chunksize % hash_block_size;
                hash_pad_len = hash_block_size - used_space_last_block;
                if (hash_pad_len >= hash_block_size)
                        hash_pad_len -= hash_block_size;
                return hash_pad_len;
        }

        used_space_last_block = chunksize % hash_block_size + 1;
        if ((hash_alg == HASH_ALG_SHA384) || (hash_alg == HASH_ALG_SHA512))
                length_len = 2 * sizeof(u64);
        else
                length_len = sizeof(u64);

        used_space_last_block += length_len;
        hash_pad_len = hash_block_size - used_space_last_block;
        if (hash_pad_len < 0)
                hash_pad_len += hash_block_size;

        hash_pad_len += 1 + length_len;
        return hash_pad_len;
}

/**
 * spum_gcm_ccm_pad_len() - Determine the required length of GCM or CCM padding.
 * @cipher_mode:        Algo type
 * @data_size:          Length of plaintext (bytes)
 *
 * Return: Length of padding, in bytes
 */
u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
                         unsigned int data_size)
{
        u32 pad_len = 0;
        u32 m1 = SPU_GCM_CCM_ALIGN - 1;

        if ((cipher_mode == CIPHER_MODE_GCM) ||
            (cipher_mode == CIPHER_MODE_CCM))
                pad_len = ((data_size + m1) & ~m1) - data_size;

        return pad_len;
}

/**
 * spum_assoc_resp_len() - Determine the size of the receive buffer required to
 * catch associated data.
 * @cipher_mode:        cipher mode
 * @assoc_len:          length of associated data (bytes)
 * @iv_len:             length of IV (bytes)
 * @is_encrypt:         true if encrypting. false if decrypting.
 *
 * Return: length of associated data in response message (bytes)
 */
u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode,
                        unsigned int assoc_len, unsigned int iv_len,
                        bool is_encrypt)
{
        u32 buflen = 0;
        u32 pad;

        if (assoc_len)
                buflen = assoc_len;

        if (cipher_mode == CIPHER_MODE_GCM) {
                /* AAD needs to be padded in responses too */
                pad = spum_gcm_ccm_pad_len(cipher_mode, buflen);
                buflen += pad;
        }
        if (cipher_mode == CIPHER_MODE_CCM) {
                /*
                 * AAD needs to be padded in responses too
                 * for CCM, len + 2 needs to be 128-bit aligned.
                 */
                pad = spum_gcm_ccm_pad_len(cipher_mode, buflen + 2);
                buflen += pad;
        }

        return buflen;
}

/**
 * spum_aead_ivlen() - Calculate the length of the AEAD IV to be included
 * in a SPU request after the AAD and before the payload.
 * @cipher_mode:  cipher mode
 * @iv_len:   initialization vector length in bytes
 *
 * In Linux ~4.2 and later, the assoc_data sg includes the IV. So no need
 * to include the IV as a separate field in the SPU request msg.
 *
 * Return: Length of AEAD IV in bytes
 */
u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len)
{
        return 0;
}

/**
 * spum_hash_type() - Determine the type of hash operation.
 * @src_sent:  The number of bytes in the current request that have already
 *             been sent to the SPU to be hashed.
 *
 * We do not use HASH_TYPE_FULL for requests that fit in a single SPU message.
 * Using FULL causes failures (such as when the string to be hashed is empty).
 * For similar reasons, we never use HASH_TYPE_FIN. Instead, submit messages
 * as INIT or UPDT and do the hash padding in sw.
 */
enum hash_type spum_hash_type(u32 src_sent)
{
        return src_sent ? HASH_TYPE_UPDT : HASH_TYPE_INIT;
}

/**
 * spum_digest_size() - Determine the size of a hash digest to expect the SPU to
 * return.
 * @alg_digest_size: Number of bytes in the final digest for the given algo
 * @alg:             The hash algorithm
 * @htype:           Type of hash operation (init, update, full, etc)
 *
 * When doing incremental hashing for an algorithm with a truncated hash
 * (e.g., SHA224), the SPU returns the full digest so that it can be fed back as
 * a partial result for the next chunk.
 */
u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg,
                     enum hash_type htype)
{
        u32 digestsize = alg_digest_size;

        /* SPU returns complete digest when doing incremental hash and truncated
         * hash algo.
         */
        if ((htype == HASH_TYPE_INIT) || (htype == HASH_TYPE_UPDT)) {
                if (alg == HASH_ALG_SHA224)
                        digestsize = SHA256_DIGEST_SIZE;
                else if (alg == HASH_ALG_SHA384)
                        digestsize = SHA512_DIGEST_SIZE;
        }
        return digestsize;
}

/**
 * spum_create_request() - Build a SPU request message header, up to and
 * including the BD header. Construct the message starting at spu_hdr. Caller
 * should allocate this buffer in DMA-able memory at least SPU_HEADER_ALLOC_LEN
 * bytes long.
 * @spu_hdr: Start of buffer where SPU request header is to be written
 * @req_opts: SPU request message options
 * @cipher_parms: Parameters related to cipher algorithm
 * @hash_parms:   Parameters related to hash algorithm
 * @aead_parms:   Parameters related to AEAD operation
 * @data_size:    Length of data to be encrypted or authenticated. If AEAD, does
 *                not include length of AAD.
 *
 * Return: the length of the SPU header in bytes. 0 if an error occurs.
 */
u32 spum_create_request(u8 *spu_hdr,
                        struct spu_request_opts *req_opts,
                        struct spu_cipher_parms *cipher_parms,
                        struct spu_hash_parms *hash_parms,
                        struct spu_aead_parms *aead_parms,
                        unsigned int data_size)
{
        struct SPUHEADER *spuh;
        struct BDESC_HEADER *bdesc;
        struct BD_HEADER *bd;

        u8 *ptr;
        u32 protocol_bits = 0;
        u32 cipher_bits = 0;
        u32 ecf_bits = 0;
        u8 sctx_words = 0;
        unsigned int buf_len = 0;

        /* size of the cipher payload */
        unsigned int cipher_len = hash_parms->prebuf_len + data_size +
                                hash_parms->pad_len;

        /* offset of prebuf or data from end of BD header */
        unsigned int cipher_offset = aead_parms->assoc_size +
                aead_parms->iv_len + aead_parms->aad_pad_len;

        /* total size of the DB data (without STAT word padding) */
        unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size,
                                                 aead_parms->iv_len,
                                                 hash_parms->prebuf_len,
                                                 data_size,
                                                 aead_parms->aad_pad_len,
                                                 aead_parms->data_pad_len,
                                                 hash_parms->pad_len);

        unsigned int auth_offset = 0;
        unsigned int offset_iv = 0;

        /* size/offset of the auth payload */
        unsigned int auth_len;

        auth_len = real_db_size;

        if (req_opts->is_aead && req_opts->is_inbound)
                cipher_len -= hash_parms->digestsize;

        if (req_opts->is_aead && req_opts->is_inbound)
                auth_len -= hash_parms->digestsize;

        if ((hash_parms->alg == HASH_ALG_AES) &&
            (hash_parms->mode == HASH_MODE_XCBC)) {
                auth_len -= hash_parms->pad_len;
                cipher_len -= hash_parms->pad_len;
        }

        flow_log("%s()\n", __func__);
        flow_log("  in:%u authFirst:%u\n",
                 req_opts->is_inbound, req_opts->auth_first);
        flow_log("  %s. cipher alg:%u mode:%u type %u\n",
                 spu_alg_name(cipher_parms->alg, cipher_parms->mode),
                 cipher_parms->alg, cipher_parms->mode, cipher_parms->type);
        flow_log("    key: %d\n", cipher_parms->key_len);
        flow_dump("    key: ", cipher_parms->key_buf, cipher_parms->key_len);
        flow_log("    iv: %d\n", cipher_parms->iv_len);
        flow_dump("    iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
        flow_log("  auth alg:%u mode:%u type %u\n",
                 hash_parms->alg, hash_parms->mode, hash_parms->type);
        flow_log("  digestsize: %u\n", hash_parms->digestsize);
        flow_log("  authkey: %d\n", hash_parms->key_len);
        flow_dump("  authkey: ", hash_parms->key_buf, hash_parms->key_len);
        flow_log("  assoc_size:%u\n", aead_parms->assoc_size);
        flow_log("  prebuf_len:%u\n", hash_parms->prebuf_len);
        flow_log("  data_size:%u\n", data_size);
        flow_log("  hash_pad_len:%u\n", hash_parms->pad_len);
        flow_log("  real_db_size:%u\n", real_db_size);
        flow_log(" auth_offset:%u auth_len:%u cipher_offset:%u cipher_len:%u\n",
                 auth_offset, auth_len, cipher_offset, cipher_len);
        flow_log("  aead_iv: %u\n", aead_parms->iv_len);

        /* starting out: zero the header (plus some) */
        ptr = spu_hdr;
        memset(ptr, 0, sizeof(struct SPUHEADER));

        /* format master header word */
        /* Do not set the next bit even though the datasheet says to */
        spuh = (struct SPUHEADER *)ptr;
        ptr += sizeof(struct SPUHEADER);
        buf_len += sizeof(struct SPUHEADER);

        spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;
        spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);

        /* Format sctx word 0 (protocol_bits) */
        sctx_words = 3;         /* size in words */

        /* Format sctx word 1 (cipher_bits) */
        if (req_opts->is_inbound)
                cipher_bits |= CIPHER_INBOUND;
        if (req_opts->auth_first)
                cipher_bits |= CIPHER_ORDER;

        /* Set the crypto parameters in the cipher.flags */
        cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;
        cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;
        cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;

        /* Set the auth parameters in the cipher.flags */
        cipher_bits |= hash_parms->alg << HASH_ALG_SHIFT;
        cipher_bits |= hash_parms->mode << HASH_MODE_SHIFT;
        cipher_bits |= hash_parms->type << HASH_TYPE_SHIFT;

        /*
         * Format sctx extensions if required, and update main fields if
         * required)
         */
        if (hash_parms->alg) {
                /* Write the authentication key material if present */
                if (hash_parms->key_len) {
                        memcpy(ptr, hash_parms->key_buf, hash_parms->key_len);
                        ptr += hash_parms->key_len;
                        buf_len += hash_parms->key_len;
                        sctx_words += hash_parms->key_len / 4;
                }

                if ((cipher_parms->mode == CIPHER_MODE_GCM) ||
                    (cipher_parms->mode == CIPHER_MODE_CCM))
                        /* unpadded length */
                        offset_iv = aead_parms->assoc_size;

                /* if GCM/CCM we need to write ICV into the payload */
                if (!req_opts->is_inbound) {
                        if ((cipher_parms->mode == CIPHER_MODE_GCM) ||
                            (cipher_parms->mode == CIPHER_MODE_CCM))
                                ecf_bits |= 1 << INSERT_ICV_SHIFT;
                } else {
                        ecf_bits |= CHECK_ICV;
                }

                /* Inform the SPU of the ICV size (in words) */
                if (hash_parms->digestsize == 64)
                        cipher_bits |= ICV_IS_512;
                else
                        ecf_bits |=
                        (hash_parms->digestsize / 4) << ICV_SIZE_SHIFT;
        }

        if (req_opts->bd_suppress)
                ecf_bits |= BD_SUPPRESS;

        /* copy the encryption keys in the SAD entry */
        if (cipher_parms->alg) {
                if (cipher_parms->key_len) {
                        memcpy(ptr, cipher_parms->key_buf,
                               cipher_parms->key_len);
                        ptr += cipher_parms->key_len;
                        buf_len += cipher_parms->key_len;
                        sctx_words += cipher_parms->key_len / 4;
                }

                /*
                 * if encrypting then set IV size, use SCTX IV unless no IV
                 * given here
                 */
                if (cipher_parms->iv_buf && cipher_parms->iv_len) {
                        /* Use SCTX IV */
                        ecf_bits |= SCTX_IV;

                        /* cipher iv provided so put it in here */
                        memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len);

                        ptr += cipher_parms->iv_len;
                        buf_len += cipher_parms->iv_len;
                        sctx_words += cipher_parms->iv_len / 4;
                }
        }

        /*
         * RFC4543 (GMAC/ESP) requires data to be sent as part of AAD
         * so we need to override the BDESC parameters.
         */
        if (req_opts->is_rfc4543) {
                if (req_opts->is_inbound)
                        data_size -= hash_parms->digestsize;
                offset_iv = aead_parms->assoc_size + data_size;
                cipher_len = 0;
                cipher_offset = offset_iv;
                auth_len = cipher_offset + aead_parms->data_pad_len;
        }

        /* write in the total sctx length now that we know it */
        protocol_bits |= sctx_words;

        /* Endian adjust the SCTX */
        spuh->sa.proto_flags = cpu_to_be32(protocol_bits);
        spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
        spuh->sa.ecf = cpu_to_be32(ecf_bits);

        /* === create the BDESC section === */
        bdesc = (struct BDESC_HEADER *)ptr;

        bdesc->offset_mac = cpu_to_be16(auth_offset);
        bdesc->length_mac = cpu_to_be16(auth_len);
        bdesc->offset_crypto = cpu_to_be16(cipher_offset);
        bdesc->length_crypto = cpu_to_be16(cipher_len);

        /*
         * CCM in SPU-M requires that ICV not be in same 32-bit word as data or
         * padding.  So account for padding as necessary.
         */
        if (cipher_parms->mode == CIPHER_MODE_CCM)
                auth_len += spum_wordalign_padlen(auth_len);

        bdesc->offset_icv = cpu_to_be16(auth_len);
        bdesc->offset_iv = cpu_to_be16(offset_iv);

        ptr += sizeof(struct BDESC_HEADER);
        buf_len += sizeof(struct BDESC_HEADER);

        /* === no MFM section === */

        /* === create the BD section === */

        /* add the BD header */
        bd = (struct BD_HEADER *)ptr;
        bd->size = cpu_to_be16(real_db_size);
        bd->prev_length = 0;

        ptr += sizeof(struct BD_HEADER);
        buf_len += sizeof(struct BD_HEADER);

        packet_dump("  SPU request header: ", spu_hdr, buf_len);

        return buf_len;
}

/**
 * spum_cipher_req_init() - Build a SPU request message header, up to and
 * including the BD header.
 * @spu_hdr:      Start of SPU request header (MH)
 * @cipher_parms: Parameters that describe the cipher request
 *
 * Construct the message starting at spu_hdr. Caller should allocate this buffer
 * in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long.
 *
 * Return: the length of the SPU header in bytes. 0 if an error occurs.
 */
u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms)
{
        struct SPUHEADER *spuh;
        u32 protocol_bits = 0;
        u32 cipher_bits = 0;
        u32 ecf_bits = 0;
        u8 sctx_words = 0;
        u8 *ptr = spu_hdr;

        flow_log("%s()\n", __func__);
        flow_log("  cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
                 cipher_parms->mode, cipher_parms->type);
        flow_log("  cipher_iv_len: %u\n", cipher_parms->iv_len);
        flow_log("    key: %d\n", cipher_parms->key_len);
        flow_dump("    key: ", cipher_parms->key_buf, cipher_parms->key_len);

        /* starting out: zero the header (plus some) */
        memset(spu_hdr, 0, sizeof(struct SPUHEADER));
        ptr += sizeof(struct SPUHEADER);

        /* format master header word */
        /* Do not set the next bit even though the datasheet says to */
        spuh = (struct SPUHEADER *)spu_hdr;

        spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;
        spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);

        /* Format sctx word 0 (protocol_bits) */
        sctx_words = 3;         /* size in words */

        /* copy the encryption keys in the SAD entry */
        if (cipher_parms->alg) {
                if (cipher_parms->key_len) {
                        ptr += cipher_parms->key_len;
                        sctx_words += cipher_parms->key_len / 4;
                }

                /*
                 * if encrypting then set IV size, use SCTX IV unless no IV
                 * given here
                 */
                if (cipher_parms->iv_len) {
                        /* Use SCTX IV */
                        ecf_bits |= SCTX_IV;
                        ptr += cipher_parms->iv_len;
                        sctx_words += cipher_parms->iv_len / 4;
                }
        }

        /* Set the crypto parameters in the cipher.flags */
        cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;
        cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;
        cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;

        /* copy the encryption keys in the SAD entry */
        if (cipher_parms->alg && cipher_parms->key_len)
                memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len);

        /* write in the total sctx length now that we know it */
        protocol_bits |= sctx_words;

        /* Endian adjust the SCTX */
        spuh->sa.proto_flags = cpu_to_be32(protocol_bits);

        /* Endian adjust the SCTX */
        spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
        spuh->sa.ecf = cpu_to_be32(ecf_bits);

        packet_dump("  SPU request header: ", spu_hdr,
                    sizeof(struct SPUHEADER));

        return sizeof(struct SPUHEADER) + cipher_parms->key_len +
                cipher_parms->iv_len + sizeof(struct BDESC_HEADER) +
                sizeof(struct BD_HEADER);
}

/**
 * spum_cipher_req_finish() - Finish building a SPU request message header for a
 * block cipher request. Assumes much of the header was already filled in at
 * setkey() time in spu_cipher_req_init().
 * @spu_hdr:         Start of the request message header (MH field)
 * @spu_req_hdr_len: Length in bytes of the SPU request header
 * @is_inbound:      0 encrypt, 1 decrypt
 * @cipher_parms:    Parameters describing cipher operation to be performed
 * @data_size:       Length of the data in the BD field
 *
 * Assumes much of the header was already filled in at setkey() time in
 * spum_cipher_req_init().
 * spum_cipher_req_init() fills in the encryption key.
 */
void spum_cipher_req_finish(u8 *spu_hdr,
                            u16 spu_req_hdr_len,
                            unsigned int is_inbound,
                            struct spu_cipher_parms *cipher_parms,
                            unsigned int data_size)
{
        struct SPUHEADER *spuh;
        struct BDESC_HEADER *bdesc;
        struct BD_HEADER *bd;
        u8 *bdesc_ptr = spu_hdr + spu_req_hdr_len -
            (sizeof(struct BD_HEADER) + sizeof(struct BDESC_HEADER));

        u32 cipher_bits;

        flow_log("%s()\n", __func__);
        flow_log(" in: %u\n", is_inbound);
        flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
                 cipher_parms->type);

        /*
         * In XTS mode, API puts "i" parameter (block tweak) in IV.  For
         * SPU-M, should be in start of the BD; tx_sg_create() copies it there.
         * IV in SPU msg for SPU-M should be 0, since that's the "j" parameter
         * (block ctr within larger data unit) - given we can send entire disk
         * block (<= 4KB) in 1 SPU msg, don't need to use this parameter.
         */
        if (cipher_parms->mode == CIPHER_MODE_XTS)
                memset(cipher_parms->iv_buf, 0, cipher_parms->iv_len);

        flow_log(" iv len: %d\n", cipher_parms->iv_len);
        flow_dump("    iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
        flow_log(" data_size: %u\n", data_size);

        /* format master header word */
        /* Do not set the next bit even though the datasheet says to */
        spuh = (struct SPUHEADER *)spu_hdr;

        /* cipher_bits was initialized at setkey time */
        cipher_bits = be32_to_cpu(spuh->sa.cipher_flags);

        /* Format sctx word 1 (cipher_bits) */
        if (is_inbound)
                cipher_bits |= CIPHER_INBOUND;
        else
                cipher_bits &= ~CIPHER_INBOUND;

        if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len)
                /* cipher iv provided so put it in here */
                memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf,
                       cipher_parms->iv_len);

        spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);

        /* === create the BDESC section === */
        bdesc = (struct BDESC_HEADER *)bdesc_ptr;
        bdesc->offset_mac = 0;
        bdesc->length_mac = 0;
        bdesc->offset_crypto = 0;

        /* XTS mode, data_size needs to include tweak parameter */
        if (cipher_parms->mode == CIPHER_MODE_XTS)
                bdesc->length_crypto = cpu_to_be16(data_size +
                                                  SPU_XTS_TWEAK_SIZE);
        else
                bdesc->length_crypto = cpu_to_be16(data_size);

        bdesc->offset_icv = 0;
        bdesc->offset_iv = 0;

        /* === no MFM section === */

        /* === create the BD section === */
        /* add the BD header */
        bd = (struct BD_HEADER *)(bdesc_ptr + sizeof(struct BDESC_HEADER));
        bd->size = cpu_to_be16(data_size);

        /* XTS mode, data_size needs to include tweak parameter */
        if (cipher_parms->mode == CIPHER_MODE_XTS)
                bd->size = cpu_to_be16(data_size + SPU_XTS_TWEAK_SIZE);

        else
                bd->size = cpu_to_be16(data_size);

        bd->prev_length = 0;

        packet_dump("  SPU request header: ", spu_hdr, spu_req_hdr_len);
}

/**
 * spum_request_pad() - Create pad bytes at the end of the data.
 * @pad_start:          Start of buffer where pad bytes are to be written
 * @gcm_ccm_padding:    length of GCM/CCM padding, in bytes
 * @hash_pad_len:       Number of bytes of padding extend data to full block
 * @auth_alg:           authentication algorithm
 * @auth_mode:          authentication mode
 * @total_sent:         length inserted at end of hash pad
 * @status_padding:     Number of bytes of padding to align STATUS word
 *
 * There may be three forms of pad:
 *  1. GCM/CCM pad - for GCM/CCM mode ciphers, pad to 16-byte alignment
 *  2. hash pad - pad to a block length, with 0x80 data terminator and
 *                size at the end
 *  3. STAT pad - to ensure the STAT field is 4-byte aligned
 */
void spum_request_pad(u8 *pad_start,
                      u32 gcm_ccm_padding,
                      u32 hash_pad_len,
                      enum hash_alg auth_alg,
                      enum hash_mode auth_mode,
                      unsigned int total_sent, u32 status_padding)
{
        u8 *ptr = pad_start;

        /* fix data alignent for GCM/CCM */
        if (gcm_ccm_padding > 0) {
                flow_log("  GCM: padding to 16 byte alignment: %u bytes\n",
                         gcm_ccm_padding);
                memset(ptr, 0, gcm_ccm_padding);
                ptr += gcm_ccm_padding;
        }

        if (hash_pad_len > 0) {
                /* clear the padding section */
                memset(ptr, 0, hash_pad_len);

                if ((auth_alg == HASH_ALG_AES) &&
                    (auth_mode == HASH_MODE_XCBC)) {
                        /* AES/XCBC just requires padding to be 0s */
                        ptr += hash_pad_len;
                } else {
                        /* terminate the data */
                        *ptr = 0x80;
                        ptr += (hash_pad_len - sizeof(u64));

                        /* add the size at the end as required per alg */
                        if (auth_alg == HASH_ALG_MD5)
                                *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull);
                        else            /* SHA1, SHA2-224, SHA2-256 */
                                *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull);
                        ptr += sizeof(u64);
                }
        }

        /* pad to a 4byte alignment for STAT */
        if (status_padding > 0) {
                flow_log("  STAT: padding to 4 byte alignment: %u bytes\n",
                         status_padding);

                memset(ptr, 0, status_padding);
                ptr += status_padding;
        }
}

/**
 * spum_xts_tweak_in_payload() - Indicate that SPUM DOES place the XTS tweak
 * field in the packet payload (rather than using IV)
 *
 * Return: 1
 */
u8 spum_xts_tweak_in_payload(void)
{
        return 1;
}

/**
 * spum_tx_status_len() - Return the length of the STATUS field in a SPU
 * response message.
 *
 * Return: Length of STATUS field in bytes.
 */
u8 spum_tx_status_len(void)
{
        return SPU_TX_STATUS_LEN;
}

/**
 * spum_rx_status_len() - Return the length of the STATUS field in a SPU
 * response message.
 *
 * Return: Length of STATUS field in bytes.
 */
u8 spum_rx_status_len(void)
{
        return SPU_RX_STATUS_LEN;
}

/**
 * spum_status_process() - Process the status from a SPU response message.
 * @statp:  start of STATUS word
 * Return:
 *   0 - if status is good and response should be processed
 *   !0 - status indicates an error and response is invalid
 */
int spum_status_process(u8 *statp)
{
        u32 status;

        status = __be32_to_cpu(*(__be32 *)statp);
        flow_log("SPU response STATUS %#08x\n", status);
        if (status & SPU_STATUS_ERROR_FLAG) {
                pr_err("%s() Warning: Error result from SPU: %#08x\n",
                       __func__, status);
                if (status & SPU_STATUS_INVALID_ICV)
                        return SPU_INVALID_ICV;
                return -EBADMSG;
        }
        return 0;
}

/**
 * spum_ccm_update_iv() - Update the IV as per the requirements for CCM mode.
 *
 * @digestsize:         Digest size of this request
 * @cipher_parms:       (pointer to) cipher parmaeters, includes IV buf & IV len
 * @assoclen:           Length of AAD data
 * @chunksize:          length of input data to be sent in this req
 * @is_encrypt:         true if this is an output/encrypt operation
 * @is_esp:             true if this is an ESP / RFC4309 operation
 *
 */
void spum_ccm_update_iv(unsigned int digestsize,
                        struct spu_cipher_parms *cipher_parms,
                        unsigned int assoclen,
                        unsigned int chunksize,
                        bool is_encrypt,
                        bool is_esp)
{
        u8 L;           /* L from CCM algorithm, length of plaintext data */
        u8 mprime;      /* M' from CCM algo, (M - 2) / 2, where M=authsize */
        u8 adata;

        if (cipher_parms->iv_len != CCM_AES_IV_SIZE) {
                pr_err("%s(): Invalid IV len %d for CCM mode, should be %d\n",
                       __func__, cipher_parms->iv_len, CCM_AES_IV_SIZE);
                return;
        }

        /*
         * IV needs to be formatted as follows:
         *
         * |          Byte 0               | Bytes 1 - N | Bytes (N+1) - 15 |
         * | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | Bits 7 - 0  |    Bits 7 - 0    |
         * | 0 |Ad?|(M - 2) / 2|   L - 1   |    Nonce    | Plaintext Length |
         *
         * Ad? = 1 if AAD present, 0 if not present
         * M = size of auth field, 8, 12, or 16 bytes (SPU-M) -or-
         *                         4, 6, 8, 10, 12, 14, 16 bytes (SPU2)
         * L = Size of Plaintext Length field; Nonce size = 15 - L
         *
         * It appears that the crypto API already expects the L-1 portion
         * to be set in the first byte of the IV, which implicitly determines
         * the nonce size, and also fills in the nonce.  But the other bits
         * in byte 0 as well as the plaintext length need to be filled in.
         *
         * In rfc4309/esp mode, L is not already in the supplied IV and
         * we need to fill it in, as well as move the IV data to be after
         * the salt
         */
        if (is_esp) {
                L = CCM_ESP_L_VALUE;    /* RFC4309 has fixed L */
        } else {
                /* L' = plaintext length - 1 so Plaintext length is L' + 1 */
                L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >>
                      CCM_B0_L_PRIME_SHIFT) + 1;
        }

        mprime = (digestsize - 2) >> 1;  /* M' = (M - 2) / 2 */
        adata = (assoclen > 0);  /* adata = 1 if any associated data */

        cipher_parms->iv_buf[0] = (adata << CCM_B0_ADATA_SHIFT) |
                                  (mprime << CCM_B0_M_PRIME_SHIFT) |
                                  ((L - 1) << CCM_B0_L_PRIME_SHIFT);

        /* Nonce is already filled in by crypto API, and is 15 - L bytes */

        /* Don't include digest in plaintext size when decrypting */
        if (!is_encrypt)
                chunksize -= digestsize;

        /* Fill in length of plaintext, formatted to be L bytes long */
        format_value_ccm(chunksize, &cipher_parms->iv_buf[15 - L + 1], L);
}

/**
 * spum_wordalign_padlen() - Given the length of a data field, determine the
 * padding required to align the data following this field on a 4-byte boundary.
 * @data_size: length of data field in bytes
 *
 * Return: length of status field padding, in bytes
 */
u32 spum_wordalign_padlen(u32 data_size)
{
        return ((data_size + 3) & ~3) - data_size;
}