LXR linux/drivers/crypto/cavium/nitrox/nitrox

   1// SPDX-License-Identifier: GPL-2.0
   2#include <linux/gfp.h>
   3#include <linux/workqueue.h>
   4#include <crypto/internal/skcipher.h>
   5
   6#include "nitrox_common.h"
   7#include "nitrox_dev.h"
   8#include "nitrox_req.h"
   9#include "nitrox_csr.h"
  10
  11/* SLC_STORE_INFO */
  12#define MIN_UDD_LEN 16
  13/* PKT_IN_HDR + SLC_STORE_INFO */
  14#define FDATA_SIZE 32
  15/* Base destination port for the solicited requests */
  16#define SOLICIT_BASE_DPORT 256
  17
  18#define REQ_NOT_POSTED 1
  19#define REQ_BACKLOG    2
  20#define REQ_POSTED     3
  21
  22/*
  23 * Response codes from SE microcode
  24 * 0x00 - Success
  25 *   Completion with no error
  26 * 0x43 - ERR_GC_DATA_LEN_INVALID
  27 *   Invalid Data length if Encryption Data length is
  28 *   less than 16 bytes for AES-XTS and AES-CTS.
  29 * 0x45 - ERR_GC_CTX_LEN_INVALID
  30 *   Invalid context length: CTXL != 23 words.
  31 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
  32 *   DOCSIS support is enabled with other than
  33 *   AES/DES-CBC mode encryption.
  34 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
  35 *   Authentication offset is other than 0 with
  36 *   Encryption IV source = 0.
  37 *   Authentication offset is other than 8 (DES)/16 (AES)
  38 *   with Encryption IV source = 1
  39 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
  40 *   CRC32 is enabled for other than DOCSIS encryption.
  41 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
  42 *   Invalid flag options in AES-CCM IV.
  43 */
  44
  45static inline int incr_index(int index, int count, int max)
  46{
  47        if ((index + count) >= max)
  48                index = index + count - max;
  49        else
  50                index += count;
  51
  52        return index;
  53}
  54
  55static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
  56{
  57        struct nitrox_device *ndev = sr->ndev;
  58        struct device *dev = DEV(ndev);
  59
  60
  61        dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg),
  62                     DMA_BIDIRECTIONAL);
  63        dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
  64                         DMA_TO_DEVICE);
  65        kfree(sr->in.sgcomp);
  66        sr->in.sg = NULL;
  67        sr->in.sgmap_cnt = 0;
  68
  69        dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg),
  70                     DMA_BIDIRECTIONAL);
  71        dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
  72                         DMA_TO_DEVICE);
  73        kfree(sr->out.sgcomp);
  74        sr->out.sg = NULL;
  75        sr->out.sgmap_cnt = 0;
  76}
  77
  78static void softreq_destroy(struct nitrox_softreq *sr)
  79{
  80        softreq_unmap_sgbufs(sr);
  81        kfree(sr);
  82}
  83
  84/**
  85 * create_sg_component - create SG componets for N5 device.
  86 * @sr: Request structure
  87 * @sgtbl: SG table
  88 * @map_nents: number of dma mapped entries
  89 *
  90 * Component structure
  91 *
  92 *   63     48 47     32 31    16 15      0
  93 *   --------------------------------------
  94 *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
  95 *   |-------------------------------------
  96 *   |               PTR0                 |
  97 *   --------------------------------------
  98 *   |               PTR1                 |
  99 *   --------------------------------------
 100 *   |               PTR2                 |
 101 *   --------------------------------------
 102 *   |               PTR3                 |
 103 *   --------------------------------------
 104 *
 105 *   Returns 0 if success or a negative errno code on error.
 106 */
 107static int create_sg_component(struct nitrox_softreq *sr,
 108                               struct nitrox_sgtable *sgtbl, int map_nents)
 109{
 110        struct nitrox_device *ndev = sr->ndev;
 111        struct nitrox_sgcomp *sgcomp;
 112        struct scatterlist *sg;
 113        dma_addr_t dma;
 114        size_t sz_comp;
 115        int i, j, nr_sgcomp;
 116
 117        nr_sgcomp = roundup(map_nents, 4) / 4;
 118
 119        /* each component holds 4 dma pointers */
 120        sz_comp = nr_sgcomp * sizeof(*sgcomp);
 121        sgcomp = kzalloc(sz_comp, sr->gfp);
 122        if (!sgcomp)
 123                return -ENOMEM;
 124
 125        sgtbl->sgcomp = sgcomp;
 126
 127        sg = sgtbl->sg;
 128        /* populate device sg component */
 129        for (i = 0; i < nr_sgcomp; i++) {
 130                for (j = 0; j < 4 && sg; j++) {
 131                        sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
 132                        sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
 133                        sg = sg_next(sg);
 134                }
 135        }
 136        /* map the device sg component */
 137        dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
 138        if (dma_mapping_error(DEV(ndev), dma)) {
 139                kfree(sgtbl->sgcomp);
 140                sgtbl->sgcomp = NULL;
 141                return -ENOMEM;
 142        }
 143
 144        sgtbl->sgcomp_dma = dma;
 145        sgtbl->sgcomp_len = sz_comp;
 146
 147        return 0;
 148}
 149
 150/**
 151 * dma_map_inbufs - DMA map input sglist and creates sglist component
 152 *                  for N5 device.
 153 * @sr: Request structure
 154 * @req: Crypto request structre
 155 *
 156 * Returns 0 if successful or a negative errno code on error.
 157 */
 158static int dma_map_inbufs(struct nitrox_softreq *sr,
 159                          struct se_crypto_request *req)
 160{
 161        struct device *dev = DEV(sr->ndev);
 162        struct scatterlist *sg;
 163        int i, nents, ret = 0;
 164
 165        nents = dma_map_sg(dev, req->src, sg_nents(req->src),
 166                           DMA_BIDIRECTIONAL);
 167        if (!nents)
 168                return -EINVAL;
 169
 170        for_each_sg(req->src, sg, nents, i)
 171                sr->in.total_bytes += sg_dma_len(sg);
 172
 173        sr->in.sg = req->src;
 174        sr->in.sgmap_cnt = nents;
 175        ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
 176        if (ret)
 177                goto incomp_err;
 178
 179        return 0;
 180
 181incomp_err:
 182        dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL);
 183        sr->in.sgmap_cnt = 0;
 184        return ret;
 185}
 186
 187static int dma_map_outbufs(struct nitrox_softreq *sr,
 188                           struct se_crypto_request *req)
 189{
 190        struct device *dev = DEV(sr->ndev);
 191        int nents, ret = 0;
 192
 193        nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
 194                           DMA_BIDIRECTIONAL);
 195        if (!nents)
 196                return -EINVAL;
 197
 198        sr->out.sg = req->dst;
 199        sr->out.sgmap_cnt = nents;
 200        ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
 201        if (ret)
 202                goto outcomp_map_err;
 203
 204        return 0;
 205
 206outcomp_map_err:
 207        dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL);
 208        sr->out.sgmap_cnt = 0;
 209        sr->out.sg = NULL;
 210        return ret;
 211}
 212
 213static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
 214                                    struct se_crypto_request *creq)
 215{
 216        int ret;
 217
 218        ret = dma_map_inbufs(sr, creq);
 219        if (ret)
 220                return ret;
 221
 222        ret = dma_map_outbufs(sr, creq);
 223        if (ret)
 224                softreq_unmap_sgbufs(sr);
 225
 226        return ret;
 227}
 228
 229static inline void backlog_list_add(struct nitrox_softreq *sr,
 230                                    struct nitrox_cmdq *cmdq)
 231{
 232        INIT_LIST_HEAD(&sr->backlog);
 233
 234        spin_lock_bh(&cmdq->backlog_qlock);
 235        list_add_tail(&sr->backlog, &cmdq->backlog_head);
 236        atomic_inc(&cmdq->backlog_count);
 237        atomic_set(&sr->status, REQ_BACKLOG);
 238        spin_unlock_bh(&cmdq->backlog_qlock);
 239}
 240
 241static inline void response_list_add(struct nitrox_softreq *sr,
 242                                     struct nitrox_cmdq *cmdq)
 243{
 244        INIT_LIST_HEAD(&sr->response);
 245
 246        spin_lock_bh(&cmdq->resp_qlock);
 247        list_add_tail(&sr->response, &cmdq->response_head);
 248        spin_unlock_bh(&cmdq->resp_qlock);
 249}
 250
 251static inline void response_list_del(struct nitrox_softreq *sr,
 252                                     struct nitrox_cmdq *cmdq)
 253{
 254        spin_lock_bh(&cmdq->resp_qlock);
 255        list_del(&sr->response);
 256        spin_unlock_bh(&cmdq->resp_qlock);
 257}
 258
 259static struct nitrox_softreq *
 260get_first_response_entry(struct nitrox_cmdq *cmdq)
 261{
 262        return list_first_entry_or_null(&cmdq->response_head,
 263                                        struct nitrox_softreq, response);
 264}
 265
 266static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
 267{
 268        if (atomic_inc_return(&cmdq->pending_count) > qlen) {
 269                atomic_dec(&cmdq->pending_count);
 270                /* sync with other cpus */
 271                smp_mb__after_atomic();
 272                return true;
 273        }
 274        /* sync with other cpus */
 275        smp_mb__after_atomic();
 276        return false;
 277}
 278
 279/**
 280 * post_se_instr - Post SE instruction to Packet Input ring
 281 * @sr: Request structure
 282 * @cmdq: Command queue structure
 283 *
 284 * Returns 0 if successful or a negative error code,
 285 * if no space in ring.
 286 */
 287static void post_se_instr(struct nitrox_softreq *sr,
 288                          struct nitrox_cmdq *cmdq)
 289{
 290        struct nitrox_device *ndev = sr->ndev;
 291        int idx;
 292        u8 *ent;
 293
 294        spin_lock_bh(&cmdq->cmd_qlock);
 295
 296        idx = cmdq->write_idx;
 297        /* copy the instruction */
 298        ent = cmdq->base + (idx * cmdq->instr_size);
 299        memcpy(ent, &sr->instr, cmdq->instr_size);
 300
 301        atomic_set(&sr->status, REQ_POSTED);
 302        response_list_add(sr, cmdq);
 303        sr->tstamp = jiffies;
 304        /* flush the command queue updates */
 305        dma_wmb();
 306
 307        /* Ring doorbell with count 1 */
 308        writeq(1, cmdq->dbell_csr_addr);
 309
 310        cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
 311
 312        spin_unlock_bh(&cmdq->cmd_qlock);
 313
 314        /* increment the posted command count */
 315        atomic64_inc(&ndev->stats.posted);
 316}
 317
 318static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
 319{
 320        struct nitrox_device *ndev = cmdq->ndev;
 321        struct nitrox_softreq *sr, *tmp;
 322        int ret = 0;
 323
 324        if (!atomic_read(&cmdq->backlog_count))
 325                return 0;
 326
 327        spin_lock_bh(&cmdq->backlog_qlock);
 328
 329        list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
 330                /* submit until space available */
 331                if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 332                        ret = -ENOSPC;
 333                        break;
 334                }
 335                /* delete from backlog list */
 336                list_del(&sr->backlog);
 337                atomic_dec(&cmdq->backlog_count);
 338                /* sync with other cpus */
 339                smp_mb__after_atomic();
 340
 341                /* post the command */
 342                post_se_instr(sr, cmdq);
 343        }
 344        spin_unlock_bh(&cmdq->backlog_qlock);
 345
 346        return ret;
 347}
 348
 349static int nitrox_enqueue_request(struct nitrox_softreq *sr)
 350{
 351        struct nitrox_cmdq *cmdq = sr->cmdq;
 352        struct nitrox_device *ndev = sr->ndev;
 353
 354        /* try to post backlog requests */
 355        post_backlog_cmds(cmdq);
 356
 357        if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 358                if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 359                        /* increment drop count */
 360                        atomic64_inc(&ndev->stats.dropped);
 361                        return -ENOSPC;
 362                }
 363                /* add to backlog list */
 364                backlog_list_add(sr, cmdq);
 365                return -EINPROGRESS;
 366        }
 367        post_se_instr(sr, cmdq);
 368
 369        return -EINPROGRESS;
 370}
 371
 372/**
 373 * nitrox_process_se_request - Send request to SE core
 374 * @ndev: NITROX device
 375 * @req: Crypto request
 376 * @callback: Completion callback
 377 * @cb_arg: Completion callback arguments
 378 *
 379 * Returns 0 on success, or a negative error code.
 380 */
 381int nitrox_process_se_request(struct nitrox_device *ndev,
 382                              struct se_crypto_request *req,
 383                              completion_t callback,
 384                              void *cb_arg)
 385{
 386        struct nitrox_softreq *sr;
 387        dma_addr_t ctx_handle = 0;
 388        int qno, ret = 0;
 389
 390        if (!nitrox_ready(ndev))
 391                return -ENODEV;
 392
 393        sr = kzalloc(sizeof(*sr), req->gfp);
 394        if (!sr)
 395                return -ENOMEM;
 396
 397        sr->ndev = ndev;
 398        sr->flags = req->flags;
 399        sr->gfp = req->gfp;
 400        sr->callback = callback;
 401        sr->cb_arg = cb_arg;
 402
 403        atomic_set(&sr->status, REQ_NOT_POSTED);
 404
 405        sr->resp.orh = req->orh;
 406        sr->resp.completion = req->comp;
 407
 408        ret = softreq_map_iobuf(sr, req);
 409        if (ret) {
 410                kfree(sr);
 411                return ret;
 412        }
 413
 414        /* get the context handle */
 415        if (req->ctx_handle) {
 416                struct ctx_hdr *hdr;
 417                u8 *ctx_ptr;
 418
 419                ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
 420                hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
 421                ctx_handle = hdr->ctx_dma;
 422        }
 423
 424        /* select the queue */
 425        qno = smp_processor_id() % ndev->nr_queues;
 426
 427        sr->cmdq = &ndev->pkt_inq[qno];
 428
 429        /*
 430         * 64-Byte Instruction Format
 431         *
 432         *  ----------------------
 433         *  |      DPTR0         | 8 bytes
 434         *  ----------------------
 435         *  |  PKT_IN_INSTR_HDR  | 8 bytes
 436         *  ----------------------
 437         *  |    PKT_IN_HDR      | 16 bytes
 438         *  ----------------------
 439         *  |    SLC_INFO        | 16 bytes
 440         *  ----------------------
 441         *  |   Front data       | 16 bytes
 442         *  ----------------------
 443         */
 444
 445        /* fill the packet instruction */
 446        /* word 0 */
 447        sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
 448
 449        /* word 1 */
 450        sr->instr.ih.value = 0;
 451        sr->instr.ih.s.g = 1;
 452        sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
 453        sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
 454        sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
 455        sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
 456        sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value);
 457
 458        /* word 2 */
 459        sr->instr.irh.value[0] = 0;
 460        sr->instr.irh.s.uddl = MIN_UDD_LEN;
 461        /* context length in 64-bit words */
 462        sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
 463        /* offset from solicit base port 256 */
 464        sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
 465        sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
 466        sr->instr.irh.s.arg = req->ctrl.s.arg;
 467        sr->instr.irh.s.opcode = req->opcode;
 468        sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]);
 469
 470        /* word 3 */
 471        sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
 472
 473        /* word 4 */
 474        sr->instr.slc.value[0] = 0;
 475        sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
 476        sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]);
 477
 478        /* word 5 */
 479        sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
 480
 481        /*
 482         * No conversion for front data,
 483         * It goes into payload
 484         * put GP Header in front data
 485         */
 486        sr->instr.fdata[0] = *((u64 *)&req->gph);
 487        sr->instr.fdata[1] = 0;
 488
 489        ret = nitrox_enqueue_request(sr);
 490        if (ret == -ENOSPC)
 491                goto send_fail;
 492
 493        return ret;
 494
 495send_fail:
 496        softreq_destroy(sr);
 497        return ret;
 498}
 499
 500static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
 501{
 502        return time_after_eq(jiffies, (tstamp + timeout));
 503}
 504
 505void backlog_qflush_work(struct work_struct *work)
 506{
 507        struct nitrox_cmdq *cmdq;
 508
 509        cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
 510        post_backlog_cmds(cmdq);
 511}
 512
 513static bool sr_completed(struct nitrox_softreq *sr)
 514{
 515        u64 orh = READ_ONCE(*sr->resp.orh);
 516        unsigned long timeout = jiffies + msecs_to_jiffies(1);
 517
 518        if ((orh != PENDING_SIG) && (orh & 0xff))
 519                return true;
 520
 521        while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
 522                if (time_after(jiffies, timeout)) {
 523                        pr_err("comp not done\n");
 524                        return false;
 525                }
 526        }
 527
 528        return true;
 529}
 530
 531/**
 532 * process_response_list - process completed requests
 533 * @cmdq: Command queue structure
 534 *
 535 * Returns the number of responses processed.
 536 */
 537static void process_response_list(struct nitrox_cmdq *cmdq)
 538{
 539        struct nitrox_device *ndev = cmdq->ndev;
 540        struct nitrox_softreq *sr;
 541        int req_completed = 0, err = 0, budget;
 542        completion_t callback;
 543        void *cb_arg;
 544
 545        /* check all pending requests */
 546        budget = atomic_read(&cmdq->pending_count);
 547
 548        while (req_completed < budget) {
 549                sr = get_first_response_entry(cmdq);
 550                if (!sr)
 551                        break;
 552
 553                if (atomic_read(&sr->status) != REQ_POSTED)
 554                        break;
 555
 556                /* check orh and completion bytes updates */
 557                if (!sr_completed(sr)) {
 558                        /* request not completed, check for timeout */
 559                        if (!cmd_timeout(sr->tstamp, ndev->timeout))
 560                                break;
 561                        dev_err_ratelimited(DEV(ndev),
 562                                            "Request timeout, orh 0x%016llx\n",
 563                                            READ_ONCE(*sr->resp.orh));
 564                }
 565                atomic_dec(&cmdq->pending_count);
 566                atomic64_inc(&ndev->stats.completed);
 567                /* sync with other cpus */
 568                smp_mb__after_atomic();
 569                /* remove from response list */
 570                response_list_del(sr, cmdq);
 571                /* ORH error code */
 572                err = READ_ONCE(*sr->resp.orh) & 0xff;
 573                callback = sr->callback;
 574                cb_arg = sr->cb_arg;
 575                softreq_destroy(sr);
 576                if (callback)
 577                        callback(cb_arg, err);
 578
 579                req_completed++;
 580        }
 581}
 582
 583/*
 584 * pkt_slc_resp_tasklet - post processing of SE responses
 585 */
 586void pkt_slc_resp_tasklet(unsigned long data)
 587{
 588        struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
 589        struct nitrox_cmdq *cmdq = qvec->cmdq;
 590        union nps_pkt_slc_cnts slc_cnts;
 591
 592        /* read completion count */
 593        slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
 594        /* resend the interrupt if more work to do */
 595        slc_cnts.s.resend = 1;
 596
 597        process_response_list(cmdq);
 598
 599        /*
 600         * clear the interrupt with resend bit enabled,
 601         * MSI-X interrupt generates if Completion count > Threshold
 602         */
 603        writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
 604
 605        if (atomic_read(&cmdq->backlog_count))
 606                schedule_work(&cmdq->backlog_qflush);
 607}
 608