LXR linux/drivers/crypto/cavium/nitrox/nitrox

   1// SPDX-License-Identifier: GPL-2.0
   2#include <linux/gfp.h>
   3#include <linux/workqueue.h>
   4#include <crypto/internal/skcipher.h>
   5
   6#include "nitrox_dev.h"
   7#include "nitrox_req.h"
   8#include "nitrox_csr.h"
   9
  10/* SLC_STORE_INFO */
  11#define MIN_UDD_LEN 16
  12/* PKT_IN_HDR + SLC_STORE_INFO */
  13#define FDATA_SIZE 32
  14/* Base destination port for the solicited requests */
  15#define SOLICIT_BASE_DPORT 256
  16
  17#define REQ_NOT_POSTED 1
  18#define REQ_BACKLOG    2
  19#define REQ_POSTED     3
  20
  21/**
  22 * Response codes from SE microcode
  23 * 0x00 - Success
  24 *   Completion with no error
  25 * 0x43 - ERR_GC_DATA_LEN_INVALID
  26 *   Invalid Data length if Encryption Data length is
  27 *   less than 16 bytes for AES-XTS and AES-CTS.
  28 * 0x45 - ERR_GC_CTX_LEN_INVALID
  29 *   Invalid context length: CTXL != 23 words.
  30 * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
  31 *   DOCSIS support is enabled with other than
  32 *   AES/DES-CBC mode encryption.
  33 * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
  34 *   Authentication offset is other than 0 with
  35 *   Encryption IV source = 0.
  36 *   Authentication offset is other than 8 (DES)/16 (AES)
  37 *   with Encryption IV source = 1
  38 * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
  39 *   CRC32 is enabled for other than DOCSIS encryption.
  40 * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
  41 *   Invalid flag options in AES-CCM IV.
  42 */
  43
  44static inline int incr_index(int index, int count, int max)
  45{
  46        if ((index + count) >= max)
  47                index = index + count - max;
  48        else
  49                index += count;
  50
  51        return index;
  52}
  53
  54static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
  55{
  56        struct nitrox_device *ndev = sr->ndev;
  57        struct device *dev = DEV(ndev);
  58
  59
  60        dma_unmap_sg(dev, sr->in.sg, sr->in.sgmap_cnt, DMA_BIDIRECTIONAL);
  61        dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
  62                         DMA_TO_DEVICE);
  63        kfree(sr->in.sgcomp);
  64        sr->in.sg = NULL;
  65        sr->in.sgmap_cnt = 0;
  66
  67        dma_unmap_sg(dev, sr->out.sg, sr->out.sgmap_cnt,
  68                     DMA_BIDIRECTIONAL);
  69        dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
  70                         DMA_TO_DEVICE);
  71        kfree(sr->out.sgcomp);
  72        sr->out.sg = NULL;
  73        sr->out.sgmap_cnt = 0;
  74}
  75
  76static void softreq_destroy(struct nitrox_softreq *sr)
  77{
  78        softreq_unmap_sgbufs(sr);
  79        kfree(sr);
  80}
  81
  82/**
  83 * create_sg_component - create SG componets for N5 device.
  84 * @sr: Request structure
  85 * @sgtbl: SG table
  86 * @map_nents: number of dma mapped entries
  87 *
  88 * Component structure
  89 *
  90 *   63     48 47     32 31    16 15      0
  91 *   --------------------------------------
  92 *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
  93 *   |-------------------------------------
  94 *   |               PTR0                 |
  95 *   --------------------------------------
  96 *   |               PTR1                 |
  97 *   --------------------------------------
  98 *   |               PTR2                 |
  99 *   --------------------------------------
 100 *   |               PTR3                 |
 101 *   --------------------------------------
 102 *
 103 *   Returns 0 if success or a negative errno code on error.
 104 */
 105static int create_sg_component(struct nitrox_softreq *sr,
 106                               struct nitrox_sgtable *sgtbl, int map_nents)
 107{
 108        struct nitrox_device *ndev = sr->ndev;
 109        struct nitrox_sgcomp *sgcomp;
 110        struct scatterlist *sg;
 111        dma_addr_t dma;
 112        size_t sz_comp;
 113        int i, j, nr_sgcomp;
 114
 115        nr_sgcomp = roundup(map_nents, 4) / 4;
 116
 117        /* each component holds 4 dma pointers */
 118        sz_comp = nr_sgcomp * sizeof(*sgcomp);
 119        sgcomp = kzalloc(sz_comp, sr->gfp);
 120        if (!sgcomp)
 121                return -ENOMEM;
 122
 123        sgtbl->sgcomp = sgcomp;
 124
 125        sg = sgtbl->sg;
 126        /* populate device sg component */
 127        for (i = 0; i < nr_sgcomp; i++) {
 128                for (j = 0; j < 4 && sg; j++) {
 129                        sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
 130                        sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
 131                        sg = sg_next(sg);
 132                }
 133        }
 134        /* map the device sg component */
 135        dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
 136        if (dma_mapping_error(DEV(ndev), dma)) {
 137                kfree(sgtbl->sgcomp);
 138                sgtbl->sgcomp = NULL;
 139                return -ENOMEM;
 140        }
 141
 142        sgtbl->sgcomp_dma = dma;
 143        sgtbl->sgcomp_len = sz_comp;
 144
 145        return 0;
 146}
 147
 148/**
 149 * dma_map_inbufs - DMA map input sglist and creates sglist component
 150 *                  for N5 device.
 151 * @sr: Request structure
 152 * @req: Crypto request structre
 153 *
 154 * Returns 0 if successful or a negative errno code on error.
 155 */
 156static int dma_map_inbufs(struct nitrox_softreq *sr,
 157                          struct se_crypto_request *req)
 158{
 159        struct device *dev = DEV(sr->ndev);
 160        struct scatterlist *sg = req->src;
 161        int i, nents, ret = 0;
 162
 163        nents = dma_map_sg(dev, req->src, sg_nents(req->src),
 164                           DMA_BIDIRECTIONAL);
 165        if (!nents)
 166                return -EINVAL;
 167
 168        for_each_sg(req->src, sg, nents, i)
 169                sr->in.total_bytes += sg_dma_len(sg);
 170
 171        sr->in.sg = req->src;
 172        sr->in.sgmap_cnt = nents;
 173        ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
 174        if (ret)
 175                goto incomp_err;
 176
 177        return 0;
 178
 179incomp_err:
 180        dma_unmap_sg(dev, req->src, nents, DMA_BIDIRECTIONAL);
 181        sr->in.sgmap_cnt = 0;
 182        return ret;
 183}
 184
 185static int dma_map_outbufs(struct nitrox_softreq *sr,
 186                           struct se_crypto_request *req)
 187{
 188        struct device *dev = DEV(sr->ndev);
 189        int nents, ret = 0;
 190
 191        nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
 192                           DMA_BIDIRECTIONAL);
 193        if (!nents)
 194                return -EINVAL;
 195
 196        sr->out.sg = req->dst;
 197        sr->out.sgmap_cnt = nents;
 198        ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
 199        if (ret)
 200                goto outcomp_map_err;
 201
 202        return 0;
 203
 204outcomp_map_err:
 205        dma_unmap_sg(dev, req->dst, nents, DMA_BIDIRECTIONAL);
 206        sr->out.sgmap_cnt = 0;
 207        sr->out.sg = NULL;
 208        return ret;
 209}
 210
 211static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
 212                                    struct se_crypto_request *creq)
 213{
 214        int ret;
 215
 216        ret = dma_map_inbufs(sr, creq);
 217        if (ret)
 218                return ret;
 219
 220        ret = dma_map_outbufs(sr, creq);
 221        if (ret)
 222                softreq_unmap_sgbufs(sr);
 223
 224        return ret;
 225}
 226
 227static inline void backlog_list_add(struct nitrox_softreq *sr,
 228                                    struct nitrox_cmdq *cmdq)
 229{
 230        INIT_LIST_HEAD(&sr->backlog);
 231
 232        spin_lock_bh(&cmdq->backlog_qlock);
 233        list_add_tail(&sr->backlog, &cmdq->backlog_head);
 234        atomic_inc(&cmdq->backlog_count);
 235        atomic_set(&sr->status, REQ_BACKLOG);
 236        spin_unlock_bh(&cmdq->backlog_qlock);
 237}
 238
 239static inline void response_list_add(struct nitrox_softreq *sr,
 240                                     struct nitrox_cmdq *cmdq)
 241{
 242        INIT_LIST_HEAD(&sr->response);
 243
 244        spin_lock_bh(&cmdq->resp_qlock);
 245        list_add_tail(&sr->response, &cmdq->response_head);
 246        spin_unlock_bh(&cmdq->resp_qlock);
 247}
 248
 249static inline void response_list_del(struct nitrox_softreq *sr,
 250                                     struct nitrox_cmdq *cmdq)
 251{
 252        spin_lock_bh(&cmdq->resp_qlock);
 253        list_del(&sr->response);
 254        spin_unlock_bh(&cmdq->resp_qlock);
 255}
 256
 257static struct nitrox_softreq *
 258get_first_response_entry(struct nitrox_cmdq *cmdq)
 259{
 260        return list_first_entry_or_null(&cmdq->response_head,
 261                                        struct nitrox_softreq, response);
 262}
 263
 264static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
 265{
 266        if (atomic_inc_return(&cmdq->pending_count) > qlen) {
 267                atomic_dec(&cmdq->pending_count);
 268                /* sync with other cpus */
 269                smp_mb__after_atomic();
 270                return true;
 271        }
 272        /* sync with other cpus */
 273        smp_mb__after_atomic();
 274        return false;
 275}
 276
 277/**
 278 * post_se_instr - Post SE instruction to Packet Input ring
 279 * @sr: Request structure
 280 *
 281 * Returns 0 if successful or a negative error code,
 282 * if no space in ring.
 283 */
 284static void post_se_instr(struct nitrox_softreq *sr,
 285                          struct nitrox_cmdq *cmdq)
 286{
 287        struct nitrox_device *ndev = sr->ndev;
 288        int idx;
 289        u8 *ent;
 290
 291        spin_lock_bh(&cmdq->cmd_qlock);
 292
 293        idx = cmdq->write_idx;
 294        /* copy the instruction */
 295        ent = cmdq->base + (idx * cmdq->instr_size);
 296        memcpy(ent, &sr->instr, cmdq->instr_size);
 297
 298        atomic_set(&sr->status, REQ_POSTED);
 299        response_list_add(sr, cmdq);
 300        sr->tstamp = jiffies;
 301        /* flush the command queue updates */
 302        dma_wmb();
 303
 304        /* Ring doorbell with count 1 */
 305        writeq(1, cmdq->dbell_csr_addr);
 306
 307        cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
 308
 309        spin_unlock_bh(&cmdq->cmd_qlock);
 310
 311        /* increment the posted command count */
 312        atomic64_inc(&ndev->stats.posted);
 313}
 314
 315static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
 316{
 317        struct nitrox_device *ndev = cmdq->ndev;
 318        struct nitrox_softreq *sr, *tmp;
 319        int ret = 0;
 320
 321        if (!atomic_read(&cmdq->backlog_count))
 322                return 0;
 323
 324        spin_lock_bh(&cmdq->backlog_qlock);
 325
 326        list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
 327                /* submit until space available */
 328                if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 329                        ret = -ENOSPC;
 330                        break;
 331                }
 332                /* delete from backlog list */
 333                list_del(&sr->backlog);
 334                atomic_dec(&cmdq->backlog_count);
 335                /* sync with other cpus */
 336                smp_mb__after_atomic();
 337
 338                /* post the command */
 339                post_se_instr(sr, cmdq);
 340        }
 341        spin_unlock_bh(&cmdq->backlog_qlock);
 342
 343        return ret;
 344}
 345
 346static int nitrox_enqueue_request(struct nitrox_softreq *sr)
 347{
 348        struct nitrox_cmdq *cmdq = sr->cmdq;
 349        struct nitrox_device *ndev = sr->ndev;
 350
 351        /* try to post backlog requests */
 352        post_backlog_cmds(cmdq);
 353
 354        if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 355                if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 356                        /* increment drop count */
 357                        atomic64_inc(&ndev->stats.dropped);
 358                        return -ENOSPC;
 359                }
 360                /* add to backlog list */
 361                backlog_list_add(sr, cmdq);
 362                return -EINPROGRESS;
 363        }
 364        post_se_instr(sr, cmdq);
 365
 366        return -EINPROGRESS;
 367}
 368
 369/**
 370 * nitrox_se_request - Send request to SE core
 371 * @ndev: NITROX device
 372 * @req: Crypto request
 373 *
 374 * Returns 0 on success, or a negative error code.
 375 */
 376int nitrox_process_se_request(struct nitrox_device *ndev,
 377                              struct se_crypto_request *req,
 378                              completion_t callback,
 379                              void *cb_arg)
 380{
 381        struct nitrox_softreq *sr;
 382        dma_addr_t ctx_handle = 0;
 383        int qno, ret = 0;
 384
 385        if (!nitrox_ready(ndev))
 386                return -ENODEV;
 387
 388        sr = kzalloc(sizeof(*sr), req->gfp);
 389        if (!sr)
 390                return -ENOMEM;
 391
 392        sr->ndev = ndev;
 393        sr->flags = req->flags;
 394        sr->gfp = req->gfp;
 395        sr->callback = callback;
 396        sr->cb_arg = cb_arg;
 397
 398        atomic_set(&sr->status, REQ_NOT_POSTED);
 399
 400        sr->resp.orh = req->orh;
 401        sr->resp.completion = req->comp;
 402
 403        ret = softreq_map_iobuf(sr, req);
 404        if (ret) {
 405                kfree(sr);
 406                return ret;
 407        }
 408
 409        /* get the context handle */
 410        if (req->ctx_handle) {
 411                struct ctx_hdr *hdr;
 412                u8 *ctx_ptr;
 413
 414                ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
 415                hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
 416                ctx_handle = hdr->ctx_dma;
 417        }
 418
 419        /* select the queue */
 420        qno = smp_processor_id() % ndev->nr_queues;
 421
 422        sr->cmdq = &ndev->pkt_inq[qno];
 423
 424        /*
 425         * 64-Byte Instruction Format
 426         *
 427         *  ----------------------
 428         *  |      DPTR0         | 8 bytes
 429         *  ----------------------
 430         *  |  PKT_IN_INSTR_HDR  | 8 bytes
 431         *  ----------------------
 432         *  |    PKT_IN_HDR      | 16 bytes
 433         *  ----------------------
 434         *  |    SLC_INFO        | 16 bytes
 435         *  ----------------------
 436         *  |   Front data       | 16 bytes
 437         *  ----------------------
 438         */
 439
 440        /* fill the packet instruction */
 441        /* word 0 */
 442        sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
 443
 444        /* word 1 */
 445        sr->instr.ih.value = 0;
 446        sr->instr.ih.s.g = 1;
 447        sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
 448        sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
 449        sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
 450        sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
 451        sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
 452
 453        /* word 2 */
 454        sr->instr.irh.value[0] = 0;
 455        sr->instr.irh.s.uddl = MIN_UDD_LEN;
 456        /* context length in 64-bit words */
 457        sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
 458        /* offset from solicit base port 256 */
 459        sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
 460        sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
 461        sr->instr.irh.s.arg = req->ctrl.s.arg;
 462        sr->instr.irh.s.opcode = req->opcode;
 463        sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
 464
 465        /* word 3 */
 466        sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
 467
 468        /* word 4 */
 469        sr->instr.slc.value[0] = 0;
 470        sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
 471        sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
 472
 473        /* word 5 */
 474        sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
 475
 476        /*
 477         * No conversion for front data,
 478         * It goes into payload
 479         * put GP Header in front data
 480         */
 481        sr->instr.fdata[0] = *((u64 *)&req->gph);
 482        sr->instr.fdata[1] = 0;
 483
 484        ret = nitrox_enqueue_request(sr);
 485        if (ret == -ENOSPC)
 486                goto send_fail;
 487
 488        return ret;
 489
 490send_fail:
 491        softreq_destroy(sr);
 492        return ret;
 493}
 494
 495static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
 496{
 497        return time_after_eq(jiffies, (tstamp + timeout));
 498}
 499
 500void backlog_qflush_work(struct work_struct *work)
 501{
 502        struct nitrox_cmdq *cmdq;
 503
 504        cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
 505        post_backlog_cmds(cmdq);
 506}
 507
 508static bool sr_completed(struct nitrox_softreq *sr)
 509{
 510        u64 orh = READ_ONCE(*sr->resp.orh);
 511        unsigned long timeout = jiffies + msecs_to_jiffies(1);
 512
 513        if ((orh != PENDING_SIG) && (orh & 0xff))
 514                return true;
 515
 516        while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
 517                if (time_after(jiffies, timeout)) {
 518                        pr_err("comp not done\n");
 519                        return false;
 520                }
 521        }
 522
 523        return true;
 524}
 525
 526/**
 527 * process_request_list - process completed requests
 528 * @ndev: N5 device
 529 * @qno: queue to operate
 530 *
 531 * Returns the number of responses processed.
 532 */
 533static void process_response_list(struct nitrox_cmdq *cmdq)
 534{
 535        struct nitrox_device *ndev = cmdq->ndev;
 536        struct nitrox_softreq *sr;
 537        int req_completed = 0, err = 0, budget;
 538        completion_t callback;
 539        void *cb_arg;
 540
 541        /* check all pending requests */
 542        budget = atomic_read(&cmdq->pending_count);
 543
 544        while (req_completed < budget) {
 545                sr = get_first_response_entry(cmdq);
 546                if (!sr)
 547                        break;
 548
 549                if (atomic_read(&sr->status) != REQ_POSTED)
 550                        break;
 551
 552                /* check orh and completion bytes updates */
 553                if (!sr_completed(sr)) {
 554                        /* request not completed, check for timeout */
 555                        if (!cmd_timeout(sr->tstamp, ndev->timeout))
 556                                break;
 557                        dev_err_ratelimited(DEV(ndev),
 558                                            "Request timeout, orh 0x%016llx\n",
 559                                            READ_ONCE(*sr->resp.orh));
 560                }
 561                atomic_dec(&cmdq->pending_count);
 562                atomic64_inc(&ndev->stats.completed);
 563                /* sync with other cpus */
 564                smp_mb__after_atomic();
 565                /* remove from response list */
 566                response_list_del(sr, cmdq);
 567                /* ORH error code */
 568                err = READ_ONCE(*sr->resp.orh) & 0xff;
 569                callback = sr->callback;
 570                cb_arg = sr->cb_arg;
 571                softreq_destroy(sr);
 572                if (callback)
 573                        callback(cb_arg, err);
 574
 575                req_completed++;
 576        }
 577}
 578
 579/**
 580 * pkt_slc_resp_tasklet - post processing of SE responses
 581 */
 582void pkt_slc_resp_tasklet(unsigned long data)
 583{
 584        struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
 585        struct nitrox_cmdq *cmdq = qvec->cmdq;
 586        union nps_pkt_slc_cnts slc_cnts;
 587
 588        /* read completion count */
 589        slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
 590        /* resend the interrupt if more work to do */
 591        slc_cnts.s.resend = 1;
 592
 593        process_response_list(cmdq);
 594
 595        /*
 596         * clear the interrupt with resend bit enabled,
 597         * MSI-X interrupt generates if Completion count > Threshold
 598         */
 599        writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
 600
 601        if (atomic_read(&cmdq->backlog_count))
 602                schedule_work(&cmdq->backlog_qflush);
 603}
 604