LXR dpdk/drivers/bus/dpaa/base/qbman/bman.h

   1/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2 *
   3 * Copyright 2010-2016 Freescale Semiconductor Inc.
   4 * Copyright 2017 NXP
   5 *
   6 */
   7
   8#ifndef __BMAN_H
   9#define __BMAN_H
  10
  11#include "bman_priv.h"
  12
  13/* Cache-inhibited register offsets */
  14#define BM_REG_RCR_PI_CINH      0x3000
  15#define BM_REG_RCR_CI_CINH      0x3100
  16#define BM_REG_RCR_ITR          0x3200
  17#define BM_REG_CFG              0x3300
  18#define BM_REG_SCN(n)           (0x3400 + ((n) << 6))
  19#define BM_REG_ISR              0x3e00
  20#define BM_REG_IIR              0x3ec0
  21
  22/* Cache-enabled register offsets */
  23#define BM_CL_CR                0x0000
  24#define BM_CL_RR0               0x0100
  25#define BM_CL_RR1               0x0140
  26#define BM_CL_RCR               0x1000
  27#define BM_CL_RCR_PI_CENA       0x3000
  28#define BM_CL_RCR_CI_CENA       0x3100
  29
  30/* BTW, the drivers (and h/w programming model) already obtain the required
  31 * synchronisation for portal accesses via lwsync(), hwsync(), and
  32 * data-dependencies. Use of barrier()s or other order-preserving primitives
  33 * simply degrade performance. Hence the use of the __raw_*() interfaces, which
  34 * simply ensure that the compiler treats the portal registers as volatile (ie.
  35 * non-coherent).
  36 */
  37
  38/* Cache-inhibited register access. */
  39#define __bm_in(bm, o)          be32_to_cpu(__raw_readl((bm)->ci + (o)))
  40#define __bm_out(bm, o, val)    __raw_writel(cpu_to_be32(val), \
  41                                             (bm)->ci + (o))
  42#define bm_in(reg)              __bm_in(&portal->addr, BM_REG_##reg)
  43#define bm_out(reg, val)        __bm_out(&portal->addr, BM_REG_##reg, val)
  44
  45/* Cache-enabled (index) register access */
  46#define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->ce + (o))
  47#define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->ce + (o))
  48#define __bm_cl_in(bm, o)       be32_to_cpu(__raw_readl((bm)->ce + (o)))
  49#define __bm_cl_out(bm, o, val) \
  50        do { \
  51                u32 *__tmpclout = (bm)->ce + (o); \
  52                __raw_writel(cpu_to_be32(val), __tmpclout); \
  53                dcbf(__tmpclout); \
  54        } while (0)
  55#define __bm_cl_invalidate(bm, o) dccivac((bm)->ce + (o))
  56#define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA)
  57#define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA)
  58#define bm_cl_in(reg)       __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA)
  59#define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val)
  60#define bm_cl_invalidate(reg)\
  61        __bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA)
  62
  63/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
  64 * analysis, look at using the "extra" bit in the ring index registers to avoid
  65 * cyclic issues.
  66 */
  67static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last)
  68{
  69        /* 'first' is included, 'last' is excluded */
  70        if (first <= last)
  71                return last - first;
  72        return ringsize + last - first;
  73}
  74
  75/* Portal modes.
  76 *   Enum types;
  77 *     pmode == production mode
  78 *     cmode == consumption mode,
  79 *   Enum values use 3 letter codes. First letter matches the portal mode,
  80 *   remaining two letters indicate;
  81 *     ci == cache-inhibited portal register
  82 *     ce == cache-enabled portal register
  83 *     vb == in-band valid-bit (cache-enabled)
  84 */
  85enum bm_rcr_pmode {             /* matches BCSP_CFG::RPM */
  86        bm_rcr_pci = 0,         /* PI index, cache-inhibited */
  87        bm_rcr_pce = 1,         /* PI index, cache-enabled */
  88        bm_rcr_pvb = 2          /* valid-bit */
  89};
  90
  91enum bm_rcr_cmode {             /* s/w-only */
  92        bm_rcr_cci,             /* CI index, cache-inhibited */
  93        bm_rcr_cce              /* CI index, cache-enabled */
  94};
  95
  96/* --- Portal structures --- */
  97
  98#define BM_RCR_SIZE             8
  99
 100struct bm_rcr {
 101        struct bm_rcr_entry *ring, *cursor;
 102        u8 ci, available, ithresh, vbit;
 103#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 104        u32 busy;
 105        enum bm_rcr_pmode pmode;
 106        enum bm_rcr_cmode cmode;
 107#endif
 108};
 109
 110struct bm_mc {
 111        struct bm_mc_command *cr;
 112        struct bm_mc_result *rr;
 113        u8 rridx, vbit;
 114#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 115        enum {
 116                /* Can only be _mc_start()ed */
 117                mc_idle,
 118                /* Can only be _mc_commit()ed or _mc_abort()ed */
 119                mc_user,
 120                /* Can only be _mc_retry()ed */
 121                mc_hw
 122        } state;
 123#endif
 124};
 125
 126struct bm_addr {
 127        void __iomem *ce;       /* cache-enabled */
 128        void __iomem *ci;       /* cache-inhibited */
 129};
 130
 131struct bm_portal {
 132        struct bm_addr addr;
 133        struct bm_rcr rcr;
 134        struct bm_mc mc;
 135        struct bm_portal_config config;
 136} ____cacheline_aligned;
 137
 138/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
 139#define RCR_CARRYCLEAR(p) \
 140        (void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6)))
 141
 142/* Bit-wise logic to convert a ring pointer to a ring index */
 143static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e)
 144{
 145        return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1);
 146}
 147
 148/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
 149static inline void RCR_INC(struct bm_rcr *rcr)
 150{
 151        /* NB: this is odd-looking, but experiments show that it generates
 152         * fast code with essentially no branching overheads. We increment to
 153         * the next RCR pointer and handle overflow and 'vbit'.
 154         */
 155        struct bm_rcr_entry *partial = rcr->cursor + 1;
 156
 157        rcr->cursor = RCR_CARRYCLEAR(partial);
 158        if (partial != rcr->cursor)
 159                rcr->vbit ^= BM_RCR_VERB_VBIT;
 160}
 161
 162static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
 163                              __maybe_unused enum bm_rcr_cmode cmode)
 164{
 165        /* This use of 'register', as well as all other occurrences, is because
 166         * it has been observed to generate much faster code with gcc than is
 167         * otherwise the case.
 168         */
 169        register struct bm_rcr *rcr = &portal->rcr;
 170        u32 cfg;
 171        u8 pi;
 172
 173        rcr->ring = portal->addr.ce + BM_CL_RCR;
 174        rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 175
 176        pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
 177        rcr->cursor = rcr->ring + pi;
 178        rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ?  BM_RCR_VERB_VBIT : 0;
 179        rcr->available = BM_RCR_SIZE - 1
 180                - bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
 181        rcr->ithresh = bm_in(RCR_ITR);
 182#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 183        rcr->busy = 0;
 184        rcr->pmode = pmode;
 185        rcr->cmode = cmode;
 186#endif
 187        cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
 188        bm_out(CFG, cfg);
 189        return 0;
 190}
 191
 192static inline void bm_rcr_finish(struct bm_portal *portal)
 193{
 194        register struct bm_rcr *rcr = &portal->rcr;
 195        u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
 196        u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 197
 198#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 199        DPAA_ASSERT(!rcr->busy);
 200#endif
 201        if (pi != RCR_PTR2IDX(rcr->cursor))
 202                pr_crit("losing uncommitted RCR entries\n");
 203        if (ci != rcr->ci)
 204                pr_crit("missing existing RCR completions\n");
 205        if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
 206                pr_crit("RCR destroyed unquiesced\n");
 207}
 208
 209static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
 210{
 211        register struct bm_rcr *rcr = &portal->rcr;
 212
 213#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 214        DPAA_ASSERT(!rcr->busy);
 215#endif
 216        if (!rcr->available)
 217                return NULL;
 218#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 219        rcr->busy = 1;
 220#endif
 221        dcbz_64(rcr->cursor);
 222        return rcr->cursor;
 223}
 224
 225static inline void bm_rcr_abort(struct bm_portal *portal)
 226{
 227        __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
 228
 229#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 230        DPAA_ASSERT(rcr->busy);
 231        rcr->busy = 0;
 232#endif
 233}
 234
 235static inline struct bm_rcr_entry *bm_rcr_pend_and_next(
 236                                        struct bm_portal *portal, u8 myverb)
 237{
 238        register struct bm_rcr *rcr = &portal->rcr;
 239
 240#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 241        DPAA_ASSERT(rcr->busy);
 242        DPAA_ASSERT(rcr->pmode != bm_rcr_pvb);
 243#endif
 244        if (rcr->available == 1)
 245                return NULL;
 246        rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
 247        dcbf_64(rcr->cursor);
 248        RCR_INC(rcr);
 249        rcr->available--;
 250        dcbz_64(rcr->cursor);
 251        return rcr->cursor;
 252}
 253
 254static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb)
 255{
 256        register struct bm_rcr *rcr = &portal->rcr;
 257
 258#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 259        DPAA_ASSERT(rcr->busy);
 260        DPAA_ASSERT(rcr->pmode == bm_rcr_pci);
 261#endif
 262        rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
 263        RCR_INC(rcr);
 264        rcr->available--;
 265        hwsync();
 266        bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
 267#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 268        rcr->busy = 0;
 269#endif
 270}
 271
 272static inline void bm_rcr_pce_prefetch(struct bm_portal *portal)
 273{
 274        __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
 275
 276#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 277        DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
 278#endif
 279        bm_cl_invalidate(RCR_PI);
 280        bm_cl_touch_rw(RCR_PI);
 281}
 282
 283static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb)
 284{
 285        register struct bm_rcr *rcr = &portal->rcr;
 286
 287#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 288        DPAA_ASSERT(rcr->busy);
 289        DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
 290#endif
 291        rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
 292        RCR_INC(rcr);
 293        rcr->available--;
 294        lwsync();
 295        bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
 296#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 297        rcr->busy = 0;
 298#endif
 299}
 300
 301static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
 302{
 303        register struct bm_rcr *rcr = &portal->rcr;
 304        struct bm_rcr_entry *rcursor;
 305
 306#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 307        DPAA_ASSERT(rcr->busy);
 308        DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
 309#endif
 310        lwsync();
 311        rcursor = rcr->cursor;
 312        rcursor->__dont_write_directly__verb = myverb | rcr->vbit;
 313        dcbf_64(rcursor);
 314        RCR_INC(rcr);
 315        rcr->available--;
 316#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 317        rcr->busy = 0;
 318#endif
 319}
 320
 321static inline u8 bm_rcr_cci_update(struct bm_portal *portal)
 322{
 323        register struct bm_rcr *rcr = &portal->rcr;
 324        u8 diff, old_ci = rcr->ci;
 325
 326#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 327        DPAA_ASSERT(rcr->cmode == bm_rcr_cci);
 328#endif
 329        rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 330        diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
 331        rcr->available += diff;
 332        return diff;
 333}
 334
 335static inline void bm_rcr_cce_prefetch(struct bm_portal *portal)
 336{
 337        __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
 338
 339#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 340        DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
 341#endif
 342        bm_cl_touch_ro(RCR_CI);
 343}
 344
 345static inline u8 bm_rcr_cce_update(struct bm_portal *portal)
 346{
 347        register struct bm_rcr *rcr = &portal->rcr;
 348        u8 diff, old_ci = rcr->ci;
 349
 350#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 351        DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
 352#endif
 353        rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1);
 354        bm_cl_invalidate(RCR_CI);
 355        diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
 356        rcr->available += diff;
 357        return diff;
 358}
 359
 360static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal)
 361{
 362        register struct bm_rcr *rcr = &portal->rcr;
 363
 364        return rcr->ithresh;
 365}
 366
 367static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
 368{
 369        register struct bm_rcr *rcr = &portal->rcr;
 370
 371        rcr->ithresh = ithresh;
 372        bm_out(RCR_ITR, ithresh);
 373}
 374
 375static inline u8 bm_rcr_get_avail(struct bm_portal *portal)
 376{
 377        register struct bm_rcr *rcr = &portal->rcr;
 378
 379        return rcr->available;
 380}
 381
 382static inline u8 bm_rcr_get_fill(struct bm_portal *portal)
 383{
 384        register struct bm_rcr *rcr = &portal->rcr;
 385
 386        return BM_RCR_SIZE - 1 - rcr->available;
 387}
 388
 389/* --- Management command API --- */
 390
 391static inline int bm_mc_init(struct bm_portal *portal)
 392{
 393        register struct bm_mc *mc = &portal->mc;
 394
 395        mc->cr = portal->addr.ce + BM_CL_CR;
 396        mc->rr = portal->addr.ce + BM_CL_RR0;
 397        mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
 398                        BM_MCC_VERB_VBIT) ?  0 : 1;
 399        mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
 400#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 401        mc->state = mc_idle;
 402#endif
 403        return 0;
 404}
 405
 406static inline void bm_mc_finish(struct bm_portal *portal)
 407{
 408        __maybe_unused register struct bm_mc *mc = &portal->mc;
 409
 410#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 411        DPAA_ASSERT(mc->state == mc_idle);
 412        if (mc->state != mc_idle)
 413                pr_crit("Losing incomplete MC command\n");
 414#endif
 415}
 416
 417static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
 418{
 419        register struct bm_mc *mc = &portal->mc;
 420
 421#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 422        DPAA_ASSERT(mc->state == mc_idle);
 423        mc->state = mc_user;
 424#endif
 425        dcbz_64(mc->cr);
 426        return mc->cr;
 427}
 428
 429static inline void bm_mc_abort(struct bm_portal *portal)
 430{
 431        __maybe_unused register struct bm_mc *mc = &portal->mc;
 432
 433#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 434        DPAA_ASSERT(mc->state == mc_user);
 435        mc->state = mc_idle;
 436#endif
 437}
 438
 439static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
 440{
 441        register struct bm_mc *mc = &portal->mc;
 442        struct bm_mc_result *rr = mc->rr + mc->rridx;
 443
 444#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 445        DPAA_ASSERT(mc->state == mc_user);
 446#endif
 447        lwsync();
 448        mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
 449        dcbf(mc->cr);
 450        dcbit_ro(rr);
 451#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 452        mc->state = mc_hw;
 453#endif
 454}
 455
 456static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
 457{
 458        register struct bm_mc *mc = &portal->mc;
 459        struct bm_mc_result *rr = mc->rr + mc->rridx;
 460
 461#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 462        DPAA_ASSERT(mc->state == mc_hw);
 463#endif
 464        /* The inactive response register's verb byte always returns zero until
 465         * its command is submitted and completed. This includes the valid-bit,
 466         * in case you were wondering.
 467         */
 468        if (!__raw_readb(&rr->verb)) {
 469                dcbit_ro(rr);
 470                return NULL;
 471        }
 472        mc->rridx ^= 1;
 473        mc->vbit ^= BM_MCC_VERB_VBIT;
 474#ifdef RTE_LIBRTE_DPAA_HWDEBUG
 475        mc->state = mc_idle;
 476#endif
 477        return rr;
 478}
 479
 480#define SCN_REG(bpid) BM_REG_SCN((bpid) / 32)
 481#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
 482static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid,
 483                                    int enable)
 484{
 485        u32 val;
 486
 487        DPAA_ASSERT(bpid < bman_pool_max);
 488        /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
 489        val = __bm_in(&portal->addr, SCN_REG(bpid));
 490        if (enable)
 491                val |= SCN_BIT(bpid);
 492        else
 493                val &= ~SCN_BIT(bpid);
 494        __bm_out(&portal->addr, SCN_REG(bpid), val);
 495}
 496
 497static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
 498{
 499#if defined(RTE_ARCH_ARM64)
 500        return __bm_in(&portal->addr, BM_REG_ISR + (n << 6));
 501#else
 502        return __bm_in(&portal->addr, BM_REG_ISR + (n << 2));
 503#endif
 504}
 505
 506static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n,
 507                                  u32 val)
 508{
 509#if defined(RTE_ARCH_ARM64)
 510        __bm_out(&portal->addr, BM_REG_ISR + (n << 6), val);
 511#else
 512        __bm_out(&portal->addr, BM_REG_ISR + (n << 2), val);
 513#endif
 514}
 515
 516/* Buffer Pool Cleanup */
 517static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid)
 518{
 519        struct bm_mc_command *bm_cmd;
 520        struct bm_mc_result *bm_res;
 521
 522        int aq_count = 0;
 523        bool stop = false;
 524
 525        while (!stop) {
 526                /* Acquire buffers until empty */
 527                bm_cmd = bm_mc_start(p);
 528                bm_cmd->acquire.bpid = bpid;
 529                bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE |  1);
 530                while (!(bm_res = bm_mc_result(p)))
 531                        cpu_relax();
 532                if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
 533                        /* Pool is empty */
 534                        stop = true;
 535                } else
 536                        ++aq_count;
 537        };
 538        return 0;
 539}
 540
 541#endif /* __BMAN_H */
 542