LXR linux/drivers/crypto/marvell/octeontx2/otx2

   1/* SPDX-License-Identifier: GPL-2.0-only
   2 * Copyright (C) 2020 Marvell.
   3 */
   4#ifndef __OTX2_CPTLF_H
   5#define __OTX2_CPTLF_H
   6
   7#include <linux/soc/marvell/octeontx2/asm.h>
   8#include <mbox.h>
   9#include <rvu.h>
  10#include "otx2_cpt_common.h"
  11#include "otx2_cpt_reqmgr.h"
  12
  13/*
  14 * CPT instruction and pending queues user requested length in CPT_INST_S msgs
  15 */
  16#define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200
  17
  18/*
  19 * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S
  20 * messages.
  21 */
  22#define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40)
  23
  24/*
  25 * CPT instruction and pending queues length in CPT_INST_S messages
  26 */
  27#define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40)
  28
  29/*
  30 * LDWB is getting incorrectly used when IQB_LDWB = 1 and CPT instruction
  31 * queue has less than 320 free entries. So, increase HW instruction queue
  32 * size by 320 and give 320 entries less for SW/NIX RX as a workaround.
  33 */
  34#define OTX2_CPT_INST_QLEN_EXTRA_BYTES  (320 * OTX2_CPT_INST_SIZE)
  35#define OTX2_CPT_EXTRA_SIZE_DIV40       (320/40)
  36
  37/* CPT instruction queue length in bytes */
  38#define OTX2_CPT_INST_QLEN_BYTES                                               \
  39                ((OTX2_CPT_SIZE_DIV40 * 40 * OTX2_CPT_INST_SIZE) +             \
  40                OTX2_CPT_INST_QLEN_EXTRA_BYTES)
  41
  42/* CPT instruction group queue length in bytes */
  43#define OTX2_CPT_INST_GRP_QLEN_BYTES                                           \
  44                ((OTX2_CPT_SIZE_DIV40 + OTX2_CPT_EXTRA_SIZE_DIV40) * 16)
  45
  46/* CPT FC length in bytes */
  47#define OTX2_CPT_Q_FC_LEN 128
  48
  49/* CPT instruction queue alignment */
  50#define OTX2_CPT_INST_Q_ALIGNMENT  128
  51
  52/* Mask which selects all engine groups */
  53#define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF
  54
  55/* Maximum LFs supported in OcteonTX2 for CPT */
  56#define OTX2_CPT_MAX_LFS_NUM    64
  57
  58/* Queue priority */
  59#define OTX2_CPT_QUEUE_HI_PRIO  0x1
  60#define OTX2_CPT_QUEUE_LOW_PRIO 0x0
  61
  62enum otx2_cptlf_state {
  63        OTX2_CPTLF_IN_RESET,
  64        OTX2_CPTLF_STARTED,
  65};
  66
  67struct otx2_cpt_inst_queue {
  68        u8 *vaddr;
  69        u8 *real_vaddr;
  70        dma_addr_t dma_addr;
  71        dma_addr_t real_dma_addr;
  72        u32 size;
  73};
  74
  75struct otx2_cptlfs_info;
  76struct otx2_cptlf_wqe {
  77        struct tasklet_struct work;
  78        struct otx2_cptlfs_info *lfs;
  79        u8 lf_num;
  80};
  81
  82struct otx2_cptlf_info {
  83        struct otx2_cptlfs_info *lfs;           /* Ptr to cptlfs_info struct */
  84        void __iomem *lmtline;                  /* Address of LMTLINE */
  85        void __iomem *ioreg;                    /* LMTLINE send register */
  86        int msix_offset;                        /* MSI-X interrupts offset */
  87        cpumask_var_t affinity_mask;            /* IRQs affinity mask */
  88        u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */
  89        u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS];  /* Is interrupt registered */
  90        u8 slot;                                /* Slot number of this LF */
  91
  92        struct otx2_cpt_inst_queue iqueue;/* Instruction queue */
  93        struct otx2_cpt_pending_queue pqueue; /* Pending queue */
  94        struct otx2_cptlf_wqe *wqe;       /* Tasklet work info */
  95};
  96
  97struct cpt_hw_ops {
  98        void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num,
  99                         struct otx2_cptlf_info *lf);
 100        u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result);
 101        u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result);
 102};
 103
 104struct otx2_cptlfs_info {
 105        /* Registers start address of VF/PF LFs are attached to */
 106        void __iomem *reg_base;
 107#define LMTLINE_SIZE  128
 108        void __iomem *lmt_base;
 109        struct pci_dev *pdev;   /* Device LFs are attached to */
 110        struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM];
 111        struct otx2_mbox *mbox;
 112        struct cpt_hw_ops *ops;
 113        u8 are_lfs_attached;    /* Whether CPT LFs are attached */
 114        u8 lfs_num;             /* Number of CPT LFs */
 115        u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */
 116        u8 kvf_limits;          /* Kernel crypto limits */
 117        atomic_t state;         /* LF's state. started/reset */
 118        int blkaddr;            /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
 119};
 120
 121static inline void otx2_cpt_free_instruction_queues(
 122                                        struct otx2_cptlfs_info *lfs)
 123{
 124        struct otx2_cpt_inst_queue *iq;
 125        int i;
 126
 127        for (i = 0; i < lfs->lfs_num; i++) {
 128                iq = &lfs->lf[i].iqueue;
 129                if (iq->real_vaddr)
 130                        dma_free_coherent(&lfs->pdev->dev,
 131                                          iq->size,
 132                                          iq->real_vaddr,
 133                                          iq->real_dma_addr);
 134                iq->real_vaddr = NULL;
 135                iq->vaddr = NULL;
 136        }
 137}
 138
 139static inline int otx2_cpt_alloc_instruction_queues(
 140                                        struct otx2_cptlfs_info *lfs)
 141{
 142        struct otx2_cpt_inst_queue *iq;
 143        int ret = 0, i;
 144
 145        if (!lfs->lfs_num)
 146                return -EINVAL;
 147
 148        for (i = 0; i < lfs->lfs_num; i++) {
 149                iq = &lfs->lf[i].iqueue;
 150                iq->size = OTX2_CPT_INST_QLEN_BYTES +
 151                           OTX2_CPT_Q_FC_LEN +
 152                           OTX2_CPT_INST_GRP_QLEN_BYTES +
 153                           OTX2_CPT_INST_Q_ALIGNMENT;
 154                iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size,
 155                                        &iq->real_dma_addr, GFP_KERNEL);
 156                if (!iq->real_vaddr) {
 157                        ret = -ENOMEM;
 158                        goto error;
 159                }
 160                iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES;
 161                iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES;
 162
 163                /* Align pointers */
 164                iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT);
 165                iq->dma_addr = PTR_ALIGN(iq->dma_addr,
 166                                         OTX2_CPT_INST_Q_ALIGNMENT);
 167        }
 168        return 0;
 169
 170error:
 171        otx2_cpt_free_instruction_queues(lfs);
 172        return ret;
 173}
 174
 175static inline void otx2_cptlf_set_iqueues_base_addr(
 176                                        struct otx2_cptlfs_info *lfs)
 177{
 178        union otx2_cptx_lf_q_base lf_q_base;
 179        int slot;
 180
 181        for (slot = 0; slot < lfs->lfs_num; slot++) {
 182                lf_q_base.u = lfs->lf[slot].iqueue.dma_addr;
 183                otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot,
 184                                 OTX2_CPT_LF_Q_BASE, lf_q_base.u);
 185        }
 186}
 187
 188static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf)
 189{
 190        union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 };
 191
 192        lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40 +
 193                                 OTX2_CPT_EXTRA_SIZE_DIV40;
 194        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 195                         OTX2_CPT_LF_Q_SIZE, lf_q_size.u);
 196}
 197
 198static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs)
 199{
 200        int slot;
 201
 202        for (slot = 0; slot < lfs->lfs_num; slot++)
 203                otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]);
 204}
 205
 206static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf)
 207{
 208        union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 };
 209        union otx2_cptx_lf_inprog lf_inprog;
 210        int timeout = 20;
 211
 212        /* Disable instructions enqueuing */
 213        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 214                         OTX2_CPT_LF_CTL, lf_ctl.u);
 215
 216        /* Wait for instruction queue to become empty */
 217        do {
 218                lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0,
 219                                              lf->slot, OTX2_CPT_LF_INPROG);
 220                if (!lf_inprog.s.inflight)
 221                        break;
 222
 223                usleep_range(10000, 20000);
 224                if (timeout-- < 0) {
 225                        dev_err(&lf->lfs->pdev->dev,
 226                                "Error LF %d is still busy.\n", lf->slot);
 227                        break;
 228                }
 229
 230        } while (1);
 231
 232        /*
 233         * Disable executions in the LF's queue,
 234         * the queue should be empty at this point
 235         */
 236        lf_inprog.s.eena = 0x0;
 237        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 238                         OTX2_CPT_LF_INPROG, lf_inprog.u);
 239}
 240
 241static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs)
 242{
 243        int slot;
 244
 245        for (slot = 0; slot < lfs->lfs_num; slot++)
 246                otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]);
 247}
 248
 249static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf,
 250                                             bool enable)
 251{
 252        union otx2_cptx_lf_ctl lf_ctl;
 253
 254        lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 255                                   OTX2_CPT_LF_CTL);
 256
 257        /* Set iqueue's enqueuing */
 258        lf_ctl.s.ena = enable ? 0x1 : 0x0;
 259        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 260                         OTX2_CPT_LF_CTL, lf_ctl.u);
 261}
 262
 263static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf)
 264{
 265        otx2_cptlf_set_iqueue_enq(lf, true);
 266}
 267
 268static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf,
 269                                              bool enable)
 270{
 271        union otx2_cptx_lf_inprog lf_inprog;
 272
 273        lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 274                                      OTX2_CPT_LF_INPROG);
 275
 276        /* Set iqueue's execution */
 277        lf_inprog.s.eena = enable ? 0x1 : 0x0;
 278        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 279                         OTX2_CPT_LF_INPROG, lf_inprog.u);
 280}
 281
 282static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf)
 283{
 284        otx2_cptlf_set_iqueue_exec(lf, true);
 285}
 286
 287static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf)
 288{
 289        otx2_cptlf_set_iqueue_exec(lf, false);
 290}
 291
 292static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs)
 293{
 294        int slot;
 295
 296        for (slot = 0; slot < lfs->lfs_num; slot++) {
 297                otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]);
 298                otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]);
 299        }
 300}
 301
 302static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst,
 303                                      struct otx2_cpt_iq_command *iq_cmd,
 304                                      u64 comp_baddr)
 305{
 306        cptinst->u[0] = 0x0;
 307        cptinst->s.doneint = true;
 308        cptinst->s.res_addr = comp_baddr;
 309        cptinst->u[2] = 0x0;
 310        cptinst->u[3] = 0x0;
 311        cptinst->s.ei0 = iq_cmd->cmd.u;
 312        cptinst->s.ei1 = iq_cmd->dptr;
 313        cptinst->s.ei2 = iq_cmd->rptr;
 314        cptinst->s.ei3 = iq_cmd->cptr.u;
 315}
 316
 317/*
 318 * On OcteonTX2 platform the parameter insts_num is used as a count of
 319 * instructions to be enqueued. The valid values for insts_num are:
 320 * 1 - 1 CPT instruction will be enqueued during LMTST operation
 321 * 2 - 2 CPT instructions will be enqueued during LMTST operation
 322 */
 323static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst,
 324                                     u32 insts_num, struct otx2_cptlf_info *lf)
 325{
 326        void __iomem *lmtline = lf->lmtline;
 327        long ret;
 328
 329        /*
 330         * Make sure memory areas pointed in CPT_INST_S
 331         * are flushed before the instruction is sent to CPT
 332         */
 333        dma_wmb();
 334
 335        do {
 336                /* Copy CPT command to LMTLINE */
 337                memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
 338
 339                /*
 340                 * LDEOR initiates atomic transfer to I/O device
 341                 * The following will cause the LMTST to fail (the LDEOR
 342                 * returns zero):
 343                 * - No stores have been performed to the LMTLINE since it was
 344                 * last invalidated.
 345                 * - The bytes which have been stored to LMTLINE since it was
 346                 * last invalidated form a pattern that is non-contiguous, does
 347                 * not start at byte 0, or does not end on a 8-byte boundary.
 348                 * (i.e.comprises a formation of other than 1–16 8-byte
 349                 * words.)
 350                 *
 351                 * These rules are designed such that an operating system
 352                 * context switch or hypervisor guest switch need have no
 353                 * knowledge of the LMTST operations; the switch code does not
 354                 * need to store to LMTCANCEL. Also note as LMTLINE data cannot
 355                 * be read, there is no information leakage between processes.
 356                 */
 357                ret = otx2_lmt_flush(lf->ioreg);
 358
 359        } while (!ret);
 360}
 361
 362static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs)
 363{
 364        return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED;
 365}
 366
 367int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri,
 368                    int lfs_num);
 369void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs);
 370int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs);
 371void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs);
 372void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs);
 373int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs);
 374
 375#endif /* __OTX2_CPTLF_H */
 376