LXR dpdk/drivers/common/cnxk/roc_nix

   1/* SPDX-License-Identifier: BSD-3-Clause
   2 * Copyright(C) 2021 Marvell.
   3 */
   4
   5#include "roc_api.h"
   6#include "roc_priv.h"
   7
   8static void
   9nix_err_intr_enb_dis(struct nix *nix, bool enb)
  10{
  11        /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */
  12        if (enb)
  13                plt_write64(~(BIT_ULL(11) | BIT_ULL(24)),
  14                            nix->base + NIX_LF_ERR_INT_ENA_W1S);
  15        else
  16                plt_write64(~0ull, nix->base + NIX_LF_ERR_INT_ENA_W1C);
  17}
  18
  19static void
  20nix_ras_intr_enb_dis(struct nix *nix, bool enb)
  21{
  22        if (enb)
  23                plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1S);
  24        else
  25                plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1C);
  26}
  27
  28void
  29roc_nix_rx_queue_intr_enable(struct roc_nix *roc_nix, uint16_t rx_queue_id)
  30{
  31        struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  32
  33        /* Enable CINT interrupt */
  34        plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1S(rx_queue_id));
  35}
  36
  37void
  38roc_nix_rx_queue_intr_disable(struct roc_nix *roc_nix, uint16_t rx_queue_id)
  39{
  40        struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  41
  42        /* Clear and disable CINT interrupt */
  43        plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(rx_queue_id));
  44}
  45
  46void
  47roc_nix_err_intr_ena_dis(struct roc_nix *roc_nix, bool enb)
  48{
  49        struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  50
  51        return nix_err_intr_enb_dis(nix, enb);
  52}
  53
  54void
  55roc_nix_ras_intr_ena_dis(struct roc_nix *roc_nix, bool enb)
  56{
  57        struct nix *nix = roc_nix_to_nix_priv(roc_nix);
  58
  59        return nix_ras_intr_enb_dis(nix, enb);
  60}
  61
  62static void
  63nix_lf_err_irq(void *param)
  64{
  65        struct nix *nix = (struct nix *)param;
  66        struct dev *dev = &nix->dev;
  67        uint64_t intr;
  68
  69        intr = plt_read64(nix->base + NIX_LF_ERR_INT);
  70        if (intr == 0)
  71                return;
  72
  73        plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
  74
  75        /* Clear interrupt */
  76        plt_write64(intr, nix->base + NIX_LF_ERR_INT);
  77        /* Dump registers to std out */
  78        roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL);
  79        roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix));
  80}
  81
  82static int
  83nix_lf_register_err_irq(struct nix *nix)
  84{
  85        struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
  86        int rc, vec;
  87
  88        vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT;
  89        /* Clear err interrupt */
  90        nix_err_intr_enb_dis(nix, false);
  91        /* Set used interrupt vectors */
  92        rc = dev_irq_register(handle, nix_lf_err_irq, nix, vec);
  93        /* Enable all dev interrupt except for RQ_DISABLED */
  94        nix_err_intr_enb_dis(nix, true);
  95
  96        return rc;
  97}
  98
  99static void
 100nix_lf_unregister_err_irq(struct nix *nix)
 101{
 102        struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
 103        int vec;
 104
 105        vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT;
 106        /* Clear err interrupt */
 107        nix_err_intr_enb_dis(nix, false);
 108        dev_irq_unregister(handle, nix_lf_err_irq, nix, vec);
 109}
 110
 111static void
 112nix_lf_ras_irq(void *param)
 113{
 114        struct nix *nix = (struct nix *)param;
 115        struct dev *dev = &nix->dev;
 116        uint64_t intr;
 117
 118        intr = plt_read64(nix->base + NIX_LF_RAS);
 119        if (intr == 0)
 120                return;
 121
 122        plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
 123        /* Clear interrupt */
 124        plt_write64(intr, nix->base + NIX_LF_RAS);
 125
 126        /* Dump registers to std out */
 127        roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL);
 128        roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix));
 129}
 130
 131static int
 132nix_lf_register_ras_irq(struct nix *nix)
 133{
 134        struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
 135        int rc, vec;
 136
 137        vec = nix->msixoff + NIX_LF_INT_VEC_POISON;
 138        /* Clear err interrupt */
 139        nix_ras_intr_enb_dis(nix, false);
 140        /* Set used interrupt vectors */
 141        rc = dev_irq_register(handle, nix_lf_ras_irq, nix, vec);
 142        /* Enable dev interrupt */
 143        nix_ras_intr_enb_dis(nix, true);
 144
 145        return rc;
 146}
 147
 148static void
 149nix_lf_unregister_ras_irq(struct nix *nix)
 150{
 151        struct plt_intr_handle *handle = &nix->pci_dev->intr_handle;
 152        int vec;
 153
 154        vec = nix->msixoff + NIX_LF_INT_VEC_POISON;
 155        /* Clear err interrupt */
 156        nix_ras_intr_enb_dis(nix, false);
 157        dev_irq_unregister(handle, nix_lf_ras_irq, nix, vec);
 158}
 159
 160static inline uint8_t
 161nix_lf_q_irq_get_and_clear(struct nix *nix, uint16_t q, uint32_t off,
 162                           uint64_t mask)
 163{
 164        uint64_t reg, wdata;
 165        uint8_t qint;
 166
 167        wdata = (uint64_t)q << 44;
 168        reg = roc_atomic64_add_nosync(wdata, (int64_t *)(nix->base + off));
 169
 170        if (reg & BIT_ULL(42) /* OP_ERR */) {
 171                plt_err("Failed execute irq get off=0x%x", off);
 172                return 0;
 173        }
 174        qint = reg & 0xff;
 175        wdata &= mask;
 176        plt_write64(wdata | qint, nix->base + off);
 177
 178        return qint;
 179}
 180
 181static inline uint8_t
 182nix_lf_rq_irq_get_and_clear(struct nix *nix, uint16_t rq)
 183{
 184        return nix_lf_q_irq_get_and_clear(nix, rq, NIX_LF_RQ_OP_INT, ~0xff00);
 185}
 186
 187static inline uint8_t
 188nix_lf_cq_irq_get_and_clear(struct nix *nix, uint16_t cq)
 189{
 190        return nix_lf_q_irq_get_and_clear(nix, cq, NIX_LF_CQ_OP_INT, ~0xff00);
 191}
 192
 193static inline uint8_t
 194nix_lf_sq_irq_get_and_clear(struct nix *nix, uint16_t sq)
 195{
 196        return nix_lf_q_irq_get_and_clear(nix, sq, NIX_LF_SQ_OP_INT, ~0x1ff00);
 197}
 198
 199static inline void
 200nix_lf_sq_debug_reg(struct nix *nix, uint32_t off)
 201{
 202        uint64_t reg;
 203
 204        reg = plt_read64(nix->base + off);
 205        if (reg & BIT_ULL(44))
 206                plt_err("SQ=%d err_code=0x%x", (int)((reg >> 8) & 0xfffff),
 207                        (uint8_t)(reg & 0xff));
 208}
 209
 210static void
 211nix_lf_cq_irq(void *param)
 212{
 213        struct nix_qint *cint = (struct nix_qint *)param;
 214        struct nix *nix = cint->nix;
 215
 216        /* Clear interrupt */
 217        plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_INT(cint->qintx));
 218}
 219
 220static void
 221nix_lf_q_irq(void *param)
 222{
 223        struct nix_qint *qint = (struct nix_qint *)param;
 224        uint8_t irq, qintx = qint->qintx;
 225        struct nix *nix = qint->nix;
 226        struct dev *dev = &nix->dev;
 227        int q, cq, rq, sq;
 228        uint64_t intr;
 229
 230        intr = plt_read64(nix->base + NIX_LF_QINTX_INT(qintx));
 231        if (intr == 0)
 232                return;
 233
 234        plt_err("Queue_intr=0x%" PRIx64 " qintx=%d pf=%d, vf=%d", intr, qintx,
 235                dev->pf, dev->vf);
 236
 237        /* Handle RQ interrupts */
 238        for (q = 0; q < nix->nb_rx_queues; q++) {
 239                rq = q % nix->qints;
 240                irq = nix_lf_rq_irq_get_and_clear(nix, rq);
 241
 242                if (irq & BIT_ULL(NIX_RQINT_DROP))
 243                        plt_err("RQ=%d NIX_RQINT_DROP", rq);
 244
 245                if (irq & BIT_ULL(NIX_RQINT_RED))
 246                        plt_err("RQ=%d NIX_RQINT_RED", rq);
 247        }
 248
 249        /* Handle CQ interrupts */
 250        for (q = 0; q < nix->nb_rx_queues; q++) {
 251                cq = q % nix->qints;
 252                irq = nix_lf_cq_irq_get_and_clear(nix, cq);
 253
 254                if (irq & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
 255                        plt_err("CQ=%d NIX_CQERRINT_DOOR_ERR", cq);
 256
 257                if (irq & BIT_ULL(NIX_CQERRINT_WR_FULL))
 258                        plt_err("CQ=%d NIX_CQERRINT_WR_FULL", cq);
 259
 260                if (irq & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
 261                        plt_err("CQ=%d NIX_CQERRINT_CQE_FAULT", cq);
 262        }
 263
 264        /* Handle SQ interrupts */
 265        for (q = 0; q < nix->nb_tx_queues; q++) {
 266                sq = q % nix->qints;
 267                irq = nix_lf_sq_irq_get_and_clear(nix, sq);
 268
 269                if (irq & BIT_ULL(NIX_SQINT_LMT_ERR)) {
 270                        plt_err("SQ=%d NIX_SQINT_LMT_ERR", sq);
 271                        nix_lf_sq_debug_reg(nix, NIX_LF_SQ_OP_ERR_DBG);
 272                }
 273                if (irq & BIT_ULL(NIX_SQINT_MNQ_ERR)) {
 274                        plt_err("SQ=%d NIX_SQINT_MNQ_ERR", sq);
 275                        nix_lf_sq_debug_reg(nix, NIX_LF_MNQ_ERR_DBG);
 276                }
 277                if (irq & BIT_ULL(NIX_SQINT_SEND_ERR)) {
 278                        plt_err("SQ=%d NIX_SQINT_SEND_ERR", sq);
 279                        nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG);
 280                }
 281                if (irq & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) {
 282                        plt_err("SQ=%d NIX_SQINT_SQB_ALLOC_FAIL", sq);
 283                        nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG);
 284                }
 285        }
 286
 287        /* Clear interrupt */
 288        plt_write64(intr, nix->base + NIX_LF_QINTX_INT(qintx));
 289
 290        /* Dump registers to std out */
 291        roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL);
 292        roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix));
 293}
 294
 295int
 296roc_nix_register_queue_irqs(struct roc_nix *roc_nix)
 297{
 298        int vec, q, sqs, rqs, qs, rc = 0;
 299        struct plt_intr_handle *handle;
 300        struct nix *nix;
 301
 302        nix = roc_nix_to_nix_priv(roc_nix);
 303        handle = &nix->pci_dev->intr_handle;
 304
 305        /* Figure out max qintx required */
 306        rqs = PLT_MIN(nix->qints, nix->nb_rx_queues);
 307        sqs = PLT_MIN(nix->qints, nix->nb_tx_queues);
 308        qs = PLT_MAX(rqs, sqs);
 309
 310        nix->configured_qints = qs;
 311
 312        nix->qints_mem =
 313                plt_zmalloc(nix->configured_qints * sizeof(struct nix_qint), 0);
 314        if (nix->qints_mem == NULL)
 315                return -ENOMEM;
 316
 317        for (q = 0; q < qs; q++) {
 318                vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q;
 319
 320                /* Clear QINT CNT */
 321                plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
 322
 323                /* Clear interrupt */
 324                plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q));
 325
 326                nix->qints_mem[q].nix = nix;
 327                nix->qints_mem[q].qintx = q;
 328
 329                /* Sync qints_mem update */
 330                plt_wmb();
 331
 332                /* Register queue irq vector */
 333                rc = dev_irq_register(handle, nix_lf_q_irq, &nix->qints_mem[q],
 334                                      vec);
 335                if (rc)
 336                        break;
 337
 338                plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
 339                plt_write64(0, nix->base + NIX_LF_QINTX_INT(q));
 340                /* Enable QINT interrupt */
 341                plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1S(q));
 342        }
 343
 344        return rc;
 345}
 346
 347void
 348roc_nix_unregister_queue_irqs(struct roc_nix *roc_nix)
 349{
 350        struct plt_intr_handle *handle;
 351        struct nix *nix;
 352        int vec, q;
 353
 354        nix = roc_nix_to_nix_priv(roc_nix);
 355        handle = &nix->pci_dev->intr_handle;
 356
 357        for (q = 0; q < nix->configured_qints; q++) {
 358                vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q;
 359
 360                /* Clear QINT CNT */
 361                plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q));
 362                plt_write64(0, nix->base + NIX_LF_QINTX_INT(q));
 363
 364                /* Clear interrupt */
 365                plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q));
 366
 367                /* Unregister queue irq vector */
 368                dev_irq_unregister(handle, nix_lf_q_irq, &nix->qints_mem[q],
 369                                   vec);
 370        }
 371        nix->configured_qints = 0;
 372
 373        plt_free(nix->qints_mem);
 374        nix->qints_mem = NULL;
 375}
 376
 377int
 378roc_nix_register_cq_irqs(struct roc_nix *roc_nix)
 379{
 380        struct plt_intr_handle *handle;
 381        uint8_t rc = 0, vec, q;
 382        struct nix *nix;
 383
 384        nix = roc_nix_to_nix_priv(roc_nix);
 385        handle = &nix->pci_dev->intr_handle;
 386
 387        nix->configured_cints = PLT_MIN(nix->cints, nix->nb_rx_queues);
 388
 389        nix->cints_mem =
 390                plt_zmalloc(nix->configured_cints * sizeof(struct nix_qint), 0);
 391        if (nix->cints_mem == NULL)
 392                return -ENOMEM;
 393
 394        for (q = 0; q < nix->configured_cints; q++) {
 395                vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q;
 396
 397                /* Clear CINT CNT */
 398                plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q));
 399
 400                /* Clear interrupt */
 401                plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q));
 402
 403                nix->cints_mem[q].nix = nix;
 404                nix->cints_mem[q].qintx = q;
 405
 406                /* Sync cints_mem update */
 407                plt_wmb();
 408
 409                /* Register queue irq vector */
 410                rc = dev_irq_register(handle, nix_lf_cq_irq, &nix->cints_mem[q],
 411                                      vec);
 412                if (rc) {
 413                        plt_err("Fail to register CQ irq, rc=%d", rc);
 414                        return rc;
 415                }
 416
 417                if (!handle->intr_vec) {
 418                        handle->intr_vec = plt_zmalloc(
 419                                nix->configured_cints * sizeof(int), 0);
 420                        if (!handle->intr_vec) {
 421                                plt_err("Failed to allocate %d rx intr_vec",
 422                                        nix->configured_cints);
 423                                return -ENOMEM;
 424                        }
 425                }
 426                /* VFIO vector zero is resereved for misc interrupt so
 427                 * doing required adjustment. (b13bfab4cd)
 428                 */
 429                handle->intr_vec[q] = PLT_INTR_VEC_RXTX_OFFSET + vec;
 430
 431                /* Configure CQE interrupt coalescing parameters */
 432                plt_write64(((CQ_CQE_THRESH_DEFAULT) |
 433                             (CQ_CQE_THRESH_DEFAULT << 32) |
 434                             (CQ_TIMER_THRESH_DEFAULT << 48)),
 435                            nix->base + NIX_LF_CINTX_WAIT((q)));
 436
 437                /* Keeping the CQ interrupt disabled as the rx interrupt
 438                 * feature needs to be enabled/disabled on demand.
 439                 */
 440        }
 441
 442        return rc;
 443}
 444
 445void
 446roc_nix_unregister_cq_irqs(struct roc_nix *roc_nix)
 447{
 448        struct plt_intr_handle *handle;
 449        struct nix *nix;
 450        int vec, q;
 451
 452        nix = roc_nix_to_nix_priv(roc_nix);
 453        handle = &nix->pci_dev->intr_handle;
 454
 455        for (q = 0; q < nix->configured_cints; q++) {
 456                vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q;
 457
 458                /* Clear CINT CNT */
 459                plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q));
 460
 461                /* Clear interrupt */
 462                plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q));
 463
 464                /* Unregister queue irq vector */
 465                dev_irq_unregister(handle, nix_lf_cq_irq, &nix->cints_mem[q],
 466                                   vec);
 467        }
 468        plt_free(nix->cints_mem);
 469}
 470
 471int
 472nix_register_irqs(struct nix *nix)
 473{
 474        int rc;
 475
 476        if (nix->msixoff == MSIX_VECTOR_INVALID) {
 477                plt_err("Invalid NIXLF MSIX vector offset vector: 0x%x",
 478                        nix->msixoff);
 479                return NIX_ERR_PARAM;
 480        }
 481
 482        /* Register lf err interrupt */
 483        rc = nix_lf_register_err_irq(nix);
 484        /* Register RAS interrupt */
 485        rc |= nix_lf_register_ras_irq(nix);
 486
 487        return rc;
 488}
 489
 490void
 491nix_unregister_irqs(struct nix *nix)
 492{
 493        nix_lf_unregister_err_irq(nix);
 494        nix_lf_unregister_ras_irq(nix);
 495}
 496