LXR dpdk/drivers/raw/octeontx2_ep/otx2_ep

   1/* SPDX-License-Identifier: BSD-3-Clause
   2 * Copyright(C) 2019 Marvell International Ltd.
   3 */
   4
   5#include <rte_common.h>
   6#include <rte_rawdev.h>
   7#include <rte_rawdev_pmd.h>
   8
   9#include "otx2_common.h"
  10#include "otx2_ep_rawdev.h"
  11#include "otx2_ep_vf.h"
  12
  13static int
  14sdp_vf_reset_iq(struct sdp_device *sdpvf, int q_no)
  15{
  16        uint64_t loop = SDP_VF_BUSY_LOOP_COUNT;
  17        volatile uint64_t d64 = 0ull;
  18
  19        /* There is no RST for a ring.
  20         * Clear all registers one by one after disabling the ring
  21         */
  22
  23        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
  24        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INSTR_BADDR(q_no));
  25        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INSTR_RSIZE(q_no));
  26
  27        d64 = 0xFFFFFFFF; /* ~0ull */
  28        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INSTR_DBELL(q_no));
  29        d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_INSTR_DBELL(q_no));
  30
  31        while ((d64 != 0) && loop--) {
  32                otx2_write64(d64, sdpvf->hw_addr +
  33                             SDP_VF_R_IN_INSTR_DBELL(q_no));
  34
  35                rte_delay_ms(1);
  36
  37                d64 = otx2_read64(sdpvf->hw_addr +
  38                                  SDP_VF_R_IN_INSTR_DBELL(q_no));
  39        }
  40
  41        loop = SDP_VF_BUSY_LOOP_COUNT;
  42        d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CNTS(q_no));
  43        while ((d64 != 0) && loop--) {
  44                otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_CNTS(q_no));
  45
  46                rte_delay_ms(1);
  47
  48                d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CNTS(q_no));
  49        }
  50
  51        d64 = 0ull;
  52        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INT_LEVELS(q_no));
  53        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_PKT_CNT(q_no));
  54        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_BYTE_CNT(q_no));
  55
  56        return 0;
  57}
  58
  59static int
  60sdp_vf_reset_oq(struct sdp_device *sdpvf, int q_no)
  61{
  62        uint64_t loop = SDP_VF_BUSY_LOOP_COUNT;
  63        volatile uint64_t d64 = 0ull;
  64
  65        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
  66
  67        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_BADDR(q_no));
  68
  69        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_RSIZE(q_no));
  70
  71        d64 = 0xFFFFFFFF;
  72        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_DBELL(q_no));
  73        d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_DBELL(q_no));
  74
  75        while ((d64 != 0) && loop--) {
  76                otx2_write64(d64, sdpvf->hw_addr +
  77                             SDP_VF_R_OUT_SLIST_DBELL(q_no));
  78
  79                rte_delay_ms(1);
  80
  81                d64 = otx2_read64(sdpvf->hw_addr +
  82                                  SDP_VF_R_OUT_SLIST_DBELL(q_no));
  83        }
  84
  85        loop = SDP_VF_BUSY_LOOP_COUNT;
  86        d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CNTS(q_no));
  87        while ((d64 != 0) && (loop--)) {
  88                otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_CNTS(q_no));
  89
  90                rte_delay_ms(1);
  91
  92                d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CNTS(q_no));
  93        }
  94
  95        d64 = 0ull;
  96        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_INT_LEVELS(q_no));
  97        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_PKT_CNT(q_no));
  98        otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_BYTE_CNT(q_no));
  99
 100        return 0;
 101}
 102
 103static void
 104sdp_vf_setup_global_iq_reg(struct sdp_device *sdpvf, int q_no)
 105{
 106        volatile uint64_t reg_val = 0ull;
 107
 108        /* Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for IQs
 109         * IS_64B is by default enabled.
 110         */
 111        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(q_no));
 112
 113        reg_val |= SDP_VF_R_IN_CTL_RDSIZE;
 114        reg_val |= SDP_VF_R_IN_CTL_IS_64B;
 115        reg_val |= SDP_VF_R_IN_CTL_ESR;
 116
 117        otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(q_no));
 118
 119}
 120
 121static void
 122sdp_vf_setup_global_oq_reg(struct sdp_device *sdpvf, int q_no)
 123{
 124        volatile uint64_t reg_val = 0ull;
 125
 126        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(q_no));
 127
 128        reg_val |= (SDP_VF_R_OUT_CTL_IMODE);
 129
 130        reg_val &= ~(SDP_VF_R_OUT_CTL_ROR_P);
 131        reg_val &= ~(SDP_VF_R_OUT_CTL_NSR_P);
 132        reg_val &= ~(SDP_VF_R_OUT_CTL_ROR_I);
 133        reg_val &= ~(SDP_VF_R_OUT_CTL_NSR_I);
 134        reg_val &= ~(SDP_VF_R_OUT_CTL_ES_I);
 135        reg_val &= ~(SDP_VF_R_OUT_CTL_ROR_D);
 136        reg_val &= ~(SDP_VF_R_OUT_CTL_NSR_D);
 137        reg_val &= ~(SDP_VF_R_OUT_CTL_ES_D);
 138
 139        /* INFO/DATA ptr swap is required  */
 140        reg_val |= (SDP_VF_R_OUT_CTL_ES_P);
 141
 142        otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(q_no));
 143
 144}
 145
 146static int
 147sdp_vf_reset_input_queues(struct sdp_device *sdpvf)
 148{
 149        uint32_t q_no = 0;
 150
 151        otx2_sdp_dbg("%s :", __func__);
 152
 153        for (q_no = 0; q_no < sdpvf->sriov_info.rings_per_vf; q_no++)
 154                sdp_vf_reset_iq(sdpvf, q_no);
 155
 156        return 0;
 157}
 158
 159static int
 160sdp_vf_reset_output_queues(struct sdp_device *sdpvf)
 161{
 162        uint64_t q_no = 0ull;
 163
 164        otx2_sdp_dbg(" %s :", __func__);
 165
 166        for (q_no = 0; q_no < sdpvf->sriov_info.rings_per_vf; q_no++)
 167                sdp_vf_reset_oq(sdpvf, q_no);
 168
 169        return 0;
 170}
 171
 172static void
 173sdp_vf_setup_global_input_regs(struct sdp_device *sdpvf)
 174{
 175        uint64_t q_no = 0ull;
 176
 177        sdp_vf_reset_input_queues(sdpvf);
 178
 179        for (q_no = 0; q_no < (sdpvf->sriov_info.rings_per_vf); q_no++)
 180                sdp_vf_setup_global_iq_reg(sdpvf, q_no);
 181}
 182
 183static void
 184sdp_vf_setup_global_output_regs(struct sdp_device *sdpvf)
 185{
 186        uint32_t q_no;
 187
 188        sdp_vf_reset_output_queues(sdpvf);
 189
 190        for (q_no = 0; q_no < (sdpvf->sriov_info.rings_per_vf); q_no++)
 191                sdp_vf_setup_global_oq_reg(sdpvf, q_no);
 192
 193}
 194
 195static int
 196sdp_vf_setup_device_regs(struct sdp_device *sdpvf)
 197{
 198        sdp_vf_setup_global_input_regs(sdpvf);
 199        sdp_vf_setup_global_output_regs(sdpvf);
 200
 201        return 0;
 202}
 203
 204static void
 205sdp_vf_setup_iq_regs(struct sdp_device *sdpvf, uint32_t iq_no)
 206{
 207        struct sdp_instr_queue *iq = sdpvf->instr_queue[iq_no];
 208        volatile uint64_t reg_val = 0ull;
 209
 210        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(iq_no));
 211
 212        /* Wait till IDLE to set to 1, not supposed to configure BADDR
 213         * as long as IDLE is 0
 214         */
 215        if (!(reg_val & SDP_VF_R_IN_CTL_IDLE)) {
 216                do {
 217                        reg_val = otx2_read64(sdpvf->hw_addr +
 218                                              SDP_VF_R_IN_CONTROL(iq_no));
 219                } while (!(reg_val & SDP_VF_R_IN_CTL_IDLE));
 220        }
 221
 222        /* Write the start of the input queue's ring and its size  */
 223        otx2_write64(iq->base_addr_dma, sdpvf->hw_addr +
 224                     SDP_VF_R_IN_INSTR_BADDR(iq_no));
 225        otx2_write64(iq->nb_desc, sdpvf->hw_addr +
 226                     SDP_VF_R_IN_INSTR_RSIZE(iq_no));
 227
 228        /* Remember the doorbell & instruction count register addr
 229         * for this queue
 230         */
 231        iq->doorbell_reg = (uint8_t *) sdpvf->hw_addr +
 232                           SDP_VF_R_IN_INSTR_DBELL(iq_no);
 233        iq->inst_cnt_reg = (uint8_t *) sdpvf->hw_addr +
 234                           SDP_VF_R_IN_CNTS(iq_no);
 235
 236        otx2_sdp_dbg("InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p",
 237                     iq_no, iq->doorbell_reg, iq->inst_cnt_reg);
 238
 239        /* Store the current instrn counter(used in flush_iq calculation) */
 240        iq->reset_instr_cnt = rte_read32(iq->inst_cnt_reg);
 241
 242        /* IN INTR_THRESHOLD is set to max(FFFFFFFF) which disable the IN INTR
 243         * to raise
 244         */
 245        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_INT_LEVELS(iq_no));
 246        reg_val = 0xffffffff;
 247
 248        otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_INT_LEVELS(iq_no));
 249
 250}
 251
 252static void
 253sdp_vf_setup_oq_regs(struct sdp_device *sdpvf, uint32_t oq_no)
 254{
 255        volatile uint64_t reg_val = 0ull;
 256        uint64_t oq_ctl = 0ull;
 257
 258        struct sdp_droq *droq = sdpvf->droq[oq_no];
 259
 260        /* Wait on IDLE to set to 1, supposed to configure BADDR
 261         * as log as IDLE is 0
 262         */
 263        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(oq_no));
 264
 265        while (!(reg_val & SDP_VF_R_OUT_CTL_IDLE)) {
 266                reg_val = otx2_read64(sdpvf->hw_addr +
 267                                      SDP_VF_R_OUT_CONTROL(oq_no));
 268        }
 269
 270        otx2_write64(droq->desc_ring_dma, sdpvf->hw_addr +
 271                     SDP_VF_R_OUT_SLIST_BADDR(oq_no));
 272        otx2_write64(droq->nb_desc, sdpvf->hw_addr +
 273                     SDP_VF_R_OUT_SLIST_RSIZE(oq_no));
 274
 275        oq_ctl = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(oq_no));
 276
 277        /* Clear the ISIZE and BSIZE (22-0) */
 278        oq_ctl &= ~(0x7fffffull);
 279
 280        /* Populate the BSIZE (15-0) */
 281        oq_ctl |= (droq->buffer_size & 0xffff);
 282
 283        /* Populate ISIZE(22-16) */
 284        oq_ctl |= ((SDP_RH_SIZE << 16) & 0x7fffff);
 285        otx2_write64(oq_ctl, sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(oq_no));
 286
 287        /* Mapped address of the pkt_sent and pkts_credit regs */
 288        droq->pkts_sent_reg = (uint8_t *) sdpvf->hw_addr +
 289                              SDP_VF_R_OUT_CNTS(oq_no);
 290        droq->pkts_credit_reg = (uint8_t *) sdpvf->hw_addr +
 291                                SDP_VF_R_OUT_SLIST_DBELL(oq_no);
 292
 293        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_INT_LEVELS(oq_no));
 294
 295        /* Clear PKT_CNT register */
 296        rte_write64(0xFFFFFFFFF, (uint8_t *)sdpvf->hw_addr +
 297                    SDP_VF_R_OUT_PKT_CNT(oq_no));
 298
 299        /* Clear the OQ doorbell  */
 300        rte_write32(0xFFFFFFFF, droq->pkts_credit_reg);
 301        while ((rte_read32(droq->pkts_credit_reg) != 0ull)) {
 302                rte_write32(0xFFFFFFFF, droq->pkts_credit_reg);
 303                rte_delay_ms(1);
 304        }
 305        otx2_sdp_dbg("SDP_R[%d]_credit:%x", oq_no,
 306                     rte_read32(droq->pkts_credit_reg));
 307
 308        /* Clear the OQ_OUT_CNTS doorbell  */
 309        reg_val = rte_read32(droq->pkts_sent_reg);
 310        rte_write32((uint32_t)reg_val, droq->pkts_sent_reg);
 311
 312        otx2_sdp_dbg("SDP_R[%d]_sent: %x", oq_no,
 313                     rte_read32(droq->pkts_sent_reg));
 314
 315        while (((rte_read32(droq->pkts_sent_reg)) != 0ull)) {
 316                reg_val = rte_read32(droq->pkts_sent_reg);
 317                rte_write32((uint32_t)reg_val, droq->pkts_sent_reg);
 318                rte_delay_ms(1);
 319        }
 320
 321}
 322
 323static void
 324sdp_vf_enable_iq(struct sdp_device *sdpvf, uint32_t q_no)
 325{
 326        volatile uint64_t reg_val = 0ull;
 327        uint64_t loop = SDP_VF_BUSY_LOOP_COUNT;
 328
 329        /* Resetting doorbells during IQ enabling also to handle abrupt
 330         * guest reboot. IQ reset does not clear the doorbells.
 331         */
 332        otx2_write64(0xFFFFFFFF, sdpvf->hw_addr +
 333                     SDP_VF_R_IN_INSTR_DBELL(q_no));
 334
 335        while (((otx2_read64(sdpvf->hw_addr +
 336                 SDP_VF_R_IN_INSTR_DBELL(q_no))) != 0ull) && loop--) {
 337
 338                rte_delay_ms(1);
 339        }
 340
 341        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 342        reg_val |= 0x1ull;
 343
 344        otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 345
 346        otx2_info("IQ[%d] enable done", q_no);
 347
 348}
 349
 350static void
 351sdp_vf_enable_oq(struct sdp_device *sdpvf, uint32_t q_no)
 352{
 353        volatile uint64_t reg_val = 0ull;
 354
 355        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 356        reg_val |= 0x1ull;
 357        otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 358
 359        otx2_info("OQ[%d] enable done", q_no);
 360}
 361
 362static void
 363sdp_vf_enable_io_queues(struct sdp_device *sdpvf)
 364{
 365        uint32_t q_no = 0;
 366
 367        for (q_no = 0; q_no < sdpvf->num_iqs; q_no++)
 368                sdp_vf_enable_iq(sdpvf, q_no);
 369
 370        for (q_no = 0; q_no < sdpvf->num_oqs; q_no++)
 371                sdp_vf_enable_oq(sdpvf, q_no);
 372}
 373
 374static void
 375sdp_vf_disable_iq(struct sdp_device *sdpvf, uint32_t q_no)
 376{
 377        volatile uint64_t reg_val = 0ull;
 378
 379        /* Reset the doorbell register for this Input Queue. */
 380        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 381        reg_val &= ~0x1ull;
 382
 383        otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 384}
 385
 386static void
 387sdp_vf_disable_oq(struct sdp_device *sdpvf, uint32_t q_no)
 388{
 389        volatile uint64_t reg_val = 0ull;
 390
 391        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 392        reg_val &= ~0x1ull;
 393
 394        otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 395
 396}
 397
 398static void
 399sdp_vf_disable_io_queues(struct sdp_device *sdpvf)
 400{
 401        uint32_t q_no = 0;
 402
 403        /* Disable Input Queues. */
 404        for (q_no = 0; q_no < sdpvf->num_iqs; q_no++)
 405                sdp_vf_disable_iq(sdpvf, q_no);
 406
 407        /* Disable Output Queues. */
 408        for (q_no = 0; q_no < sdpvf->num_oqs; q_no++)
 409                sdp_vf_disable_oq(sdpvf, q_no);
 410}
 411
 412static uint32_t
 413sdp_vf_update_read_index(struct sdp_instr_queue *iq)
 414{
 415        uint32_t new_idx = rte_read32(iq->inst_cnt_reg);
 416
 417        /* The new instr cnt reg is a 32-bit counter that can roll over.
 418         * We have noted the counter's initial value at init time into
 419         * reset_instr_cnt
 420         */
 421        if (iq->reset_instr_cnt < new_idx)
 422                new_idx -= iq->reset_instr_cnt;
 423        else
 424                new_idx += (0xffffffff - iq->reset_instr_cnt) + 1;
 425
 426        /* Modulo of the new index with the IQ size will give us
 427         * the new index.
 428         */
 429        new_idx %= iq->nb_desc;
 430
 431        return new_idx;
 432}
 433
 434int
 435sdp_vf_setup_device(struct sdp_device *sdpvf)
 436{
 437        uint64_t reg_val = 0ull;
 438
 439        /* If application doesn't provide its conf, use driver default conf */
 440        if (sdpvf->conf == NULL) {
 441                sdpvf->conf = sdp_get_defconf(sdpvf);
 442                if (sdpvf->conf == NULL) {
 443                        otx2_err("SDP VF default config not found");
 444                        return -ENOMEM;
 445                }
 446                otx2_info("Default config is used");
 447        }
 448
 449        /* Get IOQs (RPVF] count */
 450        reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(0));
 451
 452        sdpvf->sriov_info.rings_per_vf = ((reg_val >> SDP_VF_R_IN_CTL_RPVF_POS)
 453                                          & SDP_VF_R_IN_CTL_RPVF_MASK);
 454
 455        otx2_info("SDP RPVF: %d", sdpvf->sriov_info.rings_per_vf);
 456
 457        sdpvf->fn_list.setup_iq_regs       = sdp_vf_setup_iq_regs;
 458        sdpvf->fn_list.setup_oq_regs       = sdp_vf_setup_oq_regs;
 459
 460        sdpvf->fn_list.setup_device_regs   = sdp_vf_setup_device_regs;
 461        sdpvf->fn_list.update_iq_read_idx  = sdp_vf_update_read_index;
 462
 463        sdpvf->fn_list.enable_io_queues    = sdp_vf_enable_io_queues;
 464        sdpvf->fn_list.disable_io_queues   = sdp_vf_disable_io_queues;
 465
 466        sdpvf->fn_list.enable_iq           = sdp_vf_enable_iq;
 467        sdpvf->fn_list.disable_iq          = sdp_vf_disable_iq;
 468
 469        sdpvf->fn_list.enable_oq           = sdp_vf_enable_oq;
 470        sdpvf->fn_list.disable_oq          = sdp_vf_disable_oq;
 471
 472
 473        return 0;
 474
 475}
 476