LXR linux/drivers/crypto/marvell/octeontx2/otx2

   1/* SPDX-License-Identifier: GPL-2.0-only
   2 * Copyright (C) 2020 Marvell.
   3 */
   4#ifndef __OTX2_CPTLF_H
   5#define __OTX2_CPTLF_H
   6
   7#include <linux/soc/marvell/octeontx2/asm.h>
   8#include <mbox.h>
   9#include <rvu.h>
  10#include "otx2_cpt_common.h"
  11#include "otx2_cpt_reqmgr.h"
  12
  13/*
  14 * CPT instruction and pending queues user requested length in CPT_INST_S msgs
  15 */
  16#define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200
  17
  18/*
  19 * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S
  20 * messages.
  21 */
  22#define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40)
  23
  24/*
  25 * CPT instruction and pending queues length in CPT_INST_S messages
  26 */
  27#define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40)
  28
  29/* CPT instruction queue length in bytes */
  30#define OTX2_CPT_INST_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 40 * \
  31                                  OTX2_CPT_INST_SIZE)
  32
  33/* CPT instruction group queue length in bytes */
  34#define OTX2_CPT_INST_GRP_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 16)
  35
  36/* CPT FC length in bytes */
  37#define OTX2_CPT_Q_FC_LEN 128
  38
  39/* CPT instruction queue alignment */
  40#define OTX2_CPT_INST_Q_ALIGNMENT  128
  41
  42/* Mask which selects all engine groups */
  43#define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF
  44
  45/* Maximum LFs supported in OcteonTX2 for CPT */
  46#define OTX2_CPT_MAX_LFS_NUM    64
  47
  48/* Queue priority */
  49#define OTX2_CPT_QUEUE_HI_PRIO  0x1
  50#define OTX2_CPT_QUEUE_LOW_PRIO 0x0
  51
  52enum otx2_cptlf_state {
  53        OTX2_CPTLF_IN_RESET,
  54        OTX2_CPTLF_STARTED,
  55};
  56
  57struct otx2_cpt_inst_queue {
  58        u8 *vaddr;
  59        u8 *real_vaddr;
  60        dma_addr_t dma_addr;
  61        dma_addr_t real_dma_addr;
  62        u32 size;
  63};
  64
  65struct otx2_cptlfs_info;
  66struct otx2_cptlf_wqe {
  67        struct tasklet_struct work;
  68        struct otx2_cptlfs_info *lfs;
  69        u8 lf_num;
  70};
  71
  72struct otx2_cptlf_info {
  73        struct otx2_cptlfs_info *lfs;           /* Ptr to cptlfs_info struct */
  74        void __iomem *lmtline;                  /* Address of LMTLINE */
  75        void __iomem *ioreg;                    /* LMTLINE send register */
  76        int msix_offset;                        /* MSI-X interrupts offset */
  77        cpumask_var_t affinity_mask;            /* IRQs affinity mask */
  78        u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */
  79        u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS];  /* Is interrupt registered */
  80        u8 slot;                                /* Slot number of this LF */
  81
  82        struct otx2_cpt_inst_queue iqueue;/* Instruction queue */
  83        struct otx2_cpt_pending_queue pqueue; /* Pending queue */
  84        struct otx2_cptlf_wqe *wqe;       /* Tasklet work info */
  85};
  86
  87struct cpt_hw_ops {
  88        void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num,
  89                         struct otx2_cptlf_info *lf);
  90        u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result);
  91        u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result);
  92};
  93
  94struct otx2_cptlfs_info {
  95        /* Registers start address of VF/PF LFs are attached to */
  96        void __iomem *reg_base;
  97#define LMTLINE_SIZE  128
  98        void __iomem *lmt_base;
  99        struct pci_dev *pdev;   /* Device LFs are attached to */
 100        struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM];
 101        struct otx2_mbox *mbox;
 102        struct cpt_hw_ops *ops;
 103        u8 are_lfs_attached;    /* Whether CPT LFs are attached */
 104        u8 lfs_num;             /* Number of CPT LFs */
 105        u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */
 106        u8 kvf_limits;          /* Kernel crypto limits */
 107        atomic_t state;         /* LF's state. started/reset */
 108        int blkaddr;            /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
 109};
 110
 111static inline void otx2_cpt_free_instruction_queues(
 112                                        struct otx2_cptlfs_info *lfs)
 113{
 114        struct otx2_cpt_inst_queue *iq;
 115        int i;
 116
 117        for (i = 0; i < lfs->lfs_num; i++) {
 118                iq = &lfs->lf[i].iqueue;
 119                if (iq->real_vaddr)
 120                        dma_free_coherent(&lfs->pdev->dev,
 121                                          iq->size,
 122                                          iq->real_vaddr,
 123                                          iq->real_dma_addr);
 124                iq->real_vaddr = NULL;
 125                iq->vaddr = NULL;
 126        }
 127}
 128
 129static inline int otx2_cpt_alloc_instruction_queues(
 130                                        struct otx2_cptlfs_info *lfs)
 131{
 132        struct otx2_cpt_inst_queue *iq;
 133        int ret = 0, i;
 134
 135        if (!lfs->lfs_num)
 136                return -EINVAL;
 137
 138        for (i = 0; i < lfs->lfs_num; i++) {
 139                iq = &lfs->lf[i].iqueue;
 140                iq->size = OTX2_CPT_INST_QLEN_BYTES +
 141                           OTX2_CPT_Q_FC_LEN +
 142                           OTX2_CPT_INST_GRP_QLEN_BYTES +
 143                           OTX2_CPT_INST_Q_ALIGNMENT;
 144                iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size,
 145                                        &iq->real_dma_addr, GFP_KERNEL);
 146                if (!iq->real_vaddr) {
 147                        ret = -ENOMEM;
 148                        goto error;
 149                }
 150                iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES;
 151                iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES;
 152
 153                /* Align pointers */
 154                iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT);
 155                iq->dma_addr = PTR_ALIGN(iq->dma_addr,
 156                                         OTX2_CPT_INST_Q_ALIGNMENT);
 157        }
 158        return 0;
 159
 160error:
 161        otx2_cpt_free_instruction_queues(lfs);
 162        return ret;
 163}
 164
 165static inline void otx2_cptlf_set_iqueues_base_addr(
 166                                        struct otx2_cptlfs_info *lfs)
 167{
 168        union otx2_cptx_lf_q_base lf_q_base;
 169        int slot;
 170
 171        for (slot = 0; slot < lfs->lfs_num; slot++) {
 172                lf_q_base.u = lfs->lf[slot].iqueue.dma_addr;
 173                otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot,
 174                                 OTX2_CPT_LF_Q_BASE, lf_q_base.u);
 175        }
 176}
 177
 178static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf)
 179{
 180        union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 };
 181
 182        lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40;
 183        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 184                         OTX2_CPT_LF_Q_SIZE, lf_q_size.u);
 185}
 186
 187static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs)
 188{
 189        int slot;
 190
 191        for (slot = 0; slot < lfs->lfs_num; slot++)
 192                otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]);
 193}
 194
 195static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf)
 196{
 197        union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 };
 198        union otx2_cptx_lf_inprog lf_inprog;
 199        int timeout = 20;
 200
 201        /* Disable instructions enqueuing */
 202        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 203                         OTX2_CPT_LF_CTL, lf_ctl.u);
 204
 205        /* Wait for instruction queue to become empty */
 206        do {
 207                lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0,
 208                                              lf->slot, OTX2_CPT_LF_INPROG);
 209                if (!lf_inprog.s.inflight)
 210                        break;
 211
 212                usleep_range(10000, 20000);
 213                if (timeout-- < 0) {
 214                        dev_err(&lf->lfs->pdev->dev,
 215                                "Error LF %d is still busy.\n", lf->slot);
 216                        break;
 217                }
 218
 219        } while (1);
 220
 221        /*
 222         * Disable executions in the LF's queue,
 223         * the queue should be empty at this point
 224         */
 225        lf_inprog.s.eena = 0x0;
 226        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 227                         OTX2_CPT_LF_INPROG, lf_inprog.u);
 228}
 229
 230static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs)
 231{
 232        int slot;
 233
 234        for (slot = 0; slot < lfs->lfs_num; slot++)
 235                otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]);
 236}
 237
 238static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf,
 239                                             bool enable)
 240{
 241        union otx2_cptx_lf_ctl lf_ctl;
 242
 243        lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 244                                   OTX2_CPT_LF_CTL);
 245
 246        /* Set iqueue's enqueuing */
 247        lf_ctl.s.ena = enable ? 0x1 : 0x0;
 248        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 249                         OTX2_CPT_LF_CTL, lf_ctl.u);
 250}
 251
 252static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf)
 253{
 254        otx2_cptlf_set_iqueue_enq(lf, true);
 255}
 256
 257static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf,
 258                                              bool enable)
 259{
 260        union otx2_cptx_lf_inprog lf_inprog;
 261
 262        lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 263                                      OTX2_CPT_LF_INPROG);
 264
 265        /* Set iqueue's execution */
 266        lf_inprog.s.eena = enable ? 0x1 : 0x0;
 267        otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
 268                         OTX2_CPT_LF_INPROG, lf_inprog.u);
 269}
 270
 271static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf)
 272{
 273        otx2_cptlf_set_iqueue_exec(lf, true);
 274}
 275
 276static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf)
 277{
 278        otx2_cptlf_set_iqueue_exec(lf, false);
 279}
 280
 281static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs)
 282{
 283        int slot;
 284
 285        for (slot = 0; slot < lfs->lfs_num; slot++) {
 286                otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]);
 287                otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]);
 288        }
 289}
 290
 291static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst,
 292                                      struct otx2_cpt_iq_command *iq_cmd,
 293                                      u64 comp_baddr)
 294{
 295        cptinst->u[0] = 0x0;
 296        cptinst->s.doneint = true;
 297        cptinst->s.res_addr = comp_baddr;
 298        cptinst->u[2] = 0x0;
 299        cptinst->u[3] = 0x0;
 300        cptinst->s.ei0 = iq_cmd->cmd.u;
 301        cptinst->s.ei1 = iq_cmd->dptr;
 302        cptinst->s.ei2 = iq_cmd->rptr;
 303        cptinst->s.ei3 = iq_cmd->cptr.u;
 304}
 305
 306/*
 307 * On OcteonTX2 platform the parameter insts_num is used as a count of
 308 * instructions to be enqueued. The valid values for insts_num are:
 309 * 1 - 1 CPT instruction will be enqueued during LMTST operation
 310 * 2 - 2 CPT instructions will be enqueued during LMTST operation
 311 */
 312static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst,
 313                                     u32 insts_num, struct otx2_cptlf_info *lf)
 314{
 315        void __iomem *lmtline = lf->lmtline;
 316        long ret;
 317
 318        /*
 319         * Make sure memory areas pointed in CPT_INST_S
 320         * are flushed before the instruction is sent to CPT
 321         */
 322        dma_wmb();
 323
 324        do {
 325                /* Copy CPT command to LMTLINE */
 326                memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
 327
 328                /*
 329                 * LDEOR initiates atomic transfer to I/O device
 330                 * The following will cause the LMTST to fail (the LDEOR
 331                 * returns zero):
 332                 * - No stores have been performed to the LMTLINE since it was
 333                 * last invalidated.
 334                 * - The bytes which have been stored to LMTLINE since it was
 335                 * last invalidated form a pattern that is non-contiguous, does
 336                 * not start at byte 0, or does not end on a 8-byte boundary.
 337                 * (i.e.comprises a formation of other than 1–16 8-byte
 338                 * words.)
 339                 *
 340                 * These rules are designed such that an operating system
 341                 * context switch or hypervisor guest switch need have no
 342                 * knowledge of the LMTST operations; the switch code does not
 343                 * need to store to LMTCANCEL. Also note as LMTLINE data cannot
 344                 * be read, there is no information leakage between processes.
 345                 */
 346                ret = otx2_lmt_flush(lf->ioreg);
 347
 348        } while (!ret);
 349}
 350
 351static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs)
 352{
 353        return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED;
 354}
 355
 356int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri,
 357                    int lfs_num);
 358void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs);
 359int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs);
 360void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs);
 361void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs);
 362int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs);
 363
 364#endif /* __OTX2_CPTLF_H */
 365