linux/drivers/soc/fsl/qbman/bman.c
<<
>>
Prefs
   1/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
   2 *
   3 * Redistribution and use in source and binary forms, with or without
   4 * modification, are permitted provided that the following conditions are met:
   5 *     * Redistributions of source code must retain the above copyright
   6 *       notice, this list of conditions and the following disclaimer.
   7 *     * Redistributions in binary form must reproduce the above copyright
   8 *       notice, this list of conditions and the following disclaimer in the
   9 *       documentation and/or other materials provided with the distribution.
  10 *     * Neither the name of Freescale Semiconductor nor the
  11 *       names of its contributors may be used to endorse or promote products
  12 *       derived from this software without specific prior written permission.
  13 *
  14 * ALTERNATIVELY, this software may be distributed under the terms of the
  15 * GNU General Public License ("GPL") as published by the Free Software
  16 * Foundation, either version 2 of that License or (at your option) any
  17 * later version.
  18 *
  19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
  20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
  23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 */
  30
  31#include "bman_priv.h"
  32
  33#define IRQNAME         "BMan portal %d"
  34#define MAX_IRQNAME     16      /* big enough for "BMan portal %d" */
  35
  36/* Portal register assists */
  37
  38#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
  39/* Cache-inhibited register offsets */
  40#define BM_REG_RCR_PI_CINH      0x3000
  41#define BM_REG_RCR_CI_CINH      0x3100
  42#define BM_REG_RCR_ITR          0x3200
  43#define BM_REG_CFG              0x3300
  44#define BM_REG_SCN(n)           (0x3400 + ((n) << 6))
  45#define BM_REG_ISR              0x3e00
  46#define BM_REG_IER              0x3e40
  47#define BM_REG_ISDR             0x3e80
  48#define BM_REG_IIR              0x3ec0
  49
  50/* Cache-enabled register offsets */
  51#define BM_CL_CR                0x0000
  52#define BM_CL_RR0               0x0100
  53#define BM_CL_RR1               0x0140
  54#define BM_CL_RCR               0x1000
  55#define BM_CL_RCR_PI_CENA       0x3000
  56#define BM_CL_RCR_CI_CENA       0x3100
  57
  58#else
  59/* Cache-inhibited register offsets */
  60#define BM_REG_RCR_PI_CINH      0x0000
  61#define BM_REG_RCR_CI_CINH      0x0004
  62#define BM_REG_RCR_ITR          0x0008
  63#define BM_REG_CFG              0x0100
  64#define BM_REG_SCN(n)           (0x0200 + ((n) << 2))
  65#define BM_REG_ISR              0x0e00
  66#define BM_REG_IER              0x0e04
  67#define BM_REG_ISDR             0x0e08
  68#define BM_REG_IIR              0x0e0c
  69
  70/* Cache-enabled register offsets */
  71#define BM_CL_CR                0x0000
  72#define BM_CL_RR0               0x0100
  73#define BM_CL_RR1               0x0140
  74#define BM_CL_RCR               0x1000
  75#define BM_CL_RCR_PI_CENA       0x3000
  76#define BM_CL_RCR_CI_CENA       0x3100
  77#endif
  78
  79/*
  80 * Portal modes.
  81 *   Enum types;
  82 *     pmode == production mode
  83 *     cmode == consumption mode,
  84 *   Enum values use 3 letter codes. First letter matches the portal mode,
  85 *   remaining two letters indicate;
  86 *     ci == cache-inhibited portal register
  87 *     ce == cache-enabled portal register
  88 *     vb == in-band valid-bit (cache-enabled)
  89 */
  90enum bm_rcr_pmode {             /* matches BCSP_CFG::RPM */
  91        bm_rcr_pci = 0,         /* PI index, cache-inhibited */
  92        bm_rcr_pce = 1,         /* PI index, cache-enabled */
  93        bm_rcr_pvb = 2          /* valid-bit */
  94};
  95enum bm_rcr_cmode {             /* s/w-only */
  96        bm_rcr_cci,             /* CI index, cache-inhibited */
  97        bm_rcr_cce              /* CI index, cache-enabled */
  98};
  99
 100
 101/* --- Portal structures --- */
 102
 103#define BM_RCR_SIZE             8
 104
 105/* Release Command */
 106struct bm_rcr_entry {
 107        union {
 108                struct {
 109                        u8 _ncw_verb; /* writes to this are non-coherent */
 110                        u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */
 111                        u8 __reserved1[62];
 112                };
 113                struct bm_buffer bufs[8];
 114        };
 115};
 116#define BM_RCR_VERB_VBIT                0x80
 117#define BM_RCR_VERB_CMD_MASK            0x70    /* one of two values; */
 118#define BM_RCR_VERB_CMD_BPID_SINGLE     0x20
 119#define BM_RCR_VERB_CMD_BPID_MULTI      0x30
 120#define BM_RCR_VERB_BUFCOUNT_MASK       0x0f    /* values 1..8 */
 121
 122struct bm_rcr {
 123        struct bm_rcr_entry *ring, *cursor;
 124        u8 ci, available, ithresh, vbit;
 125#ifdef CONFIG_FSL_DPAA_CHECKING
 126        u32 busy;
 127        enum bm_rcr_pmode pmode;
 128        enum bm_rcr_cmode cmode;
 129#endif
 130};
 131
 132/* MC (Management Command) command */
 133struct bm_mc_command {
 134        u8 _ncw_verb; /* writes to this are non-coherent */
 135        u8 bpid; /* used by acquire command */
 136        u8 __reserved[62];
 137};
 138#define BM_MCC_VERB_VBIT                0x80
 139#define BM_MCC_VERB_CMD_MASK            0x70    /* where the verb contains; */
 140#define BM_MCC_VERB_CMD_ACQUIRE         0x10
 141#define BM_MCC_VERB_CMD_QUERY           0x40
 142#define BM_MCC_VERB_ACQUIRE_BUFCOUNT    0x0f    /* values 1..8 go here */
 143
 144/* MC result, Acquire and Query Response */
 145union bm_mc_result {
 146        struct {
 147                u8 verb;
 148                u8 bpid;
 149                u8 __reserved[62];
 150        };
 151        struct bm_buffer bufs[8];
 152};
 153#define BM_MCR_VERB_VBIT                0x80
 154#define BM_MCR_VERB_CMD_MASK            BM_MCC_VERB_CMD_MASK
 155#define BM_MCR_VERB_CMD_ACQUIRE         BM_MCC_VERB_CMD_ACQUIRE
 156#define BM_MCR_VERB_CMD_QUERY           BM_MCC_VERB_CMD_QUERY
 157#define BM_MCR_VERB_CMD_ERR_INVALID     0x60
 158#define BM_MCR_VERB_CMD_ERR_ECC         0x70
 159#define BM_MCR_VERB_ACQUIRE_BUFCOUNT    BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */
 160#define BM_MCR_TIMEOUT                  10000 /* us */
 161
 162struct bm_mc {
 163        struct bm_mc_command *cr;
 164        union bm_mc_result *rr;
 165        u8 rridx, vbit;
 166#ifdef CONFIG_FSL_DPAA_CHECKING
 167        enum {
 168                /* Can only be _mc_start()ed */
 169                mc_idle,
 170                /* Can only be _mc_commit()ed or _mc_abort()ed */
 171                mc_user,
 172                /* Can only be _mc_retry()ed */
 173                mc_hw
 174        } state;
 175#endif
 176};
 177
 178struct bm_addr {
 179        void *ce;               /* cache-enabled */
 180        __be32 *ce_be;          /* Same as above but for direct access */
 181        void __iomem *ci;       /* cache-inhibited */
 182};
 183
 184struct bm_portal {
 185        struct bm_addr addr;
 186        struct bm_rcr rcr;
 187        struct bm_mc mc;
 188} ____cacheline_aligned;
 189
 190/* Cache-inhibited register access. */
 191static inline u32 bm_in(struct bm_portal *p, u32 offset)
 192{
 193        return ioread32be(p->addr.ci + offset);
 194}
 195
 196static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)
 197{
 198        iowrite32be(val, p->addr.ci + offset);
 199}
 200
 201/* Cache Enabled Portal Access */
 202static inline void bm_cl_invalidate(struct bm_portal *p, u32 offset)
 203{
 204        dpaa_invalidate(p->addr.ce + offset);
 205}
 206
 207static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)
 208{
 209        dpaa_touch_ro(p->addr.ce + offset);
 210}
 211
 212static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)
 213{
 214        return be32_to_cpu(*(p->addr.ce_be + (offset/4)));
 215}
 216
 217struct bman_portal {
 218        struct bm_portal p;
 219        /* interrupt sources processed by portal_isr(), configurable */
 220        unsigned long irq_sources;
 221        /* probing time config params for cpu-affine portals */
 222        const struct bm_portal_config *config;
 223        char irqname[MAX_IRQNAME];
 224};
 225
 226static cpumask_t affine_mask;
 227static DEFINE_SPINLOCK(affine_mask_lock);
 228static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
 229
 230static inline struct bman_portal *get_affine_portal(void)
 231{
 232        return &get_cpu_var(bman_affine_portal);
 233}
 234
 235static inline void put_affine_portal(void)
 236{
 237        put_cpu_var(bman_affine_portal);
 238}
 239
 240/*
 241 * This object type refers to a pool, it isn't *the* pool. There may be
 242 * more than one such object per BMan buffer pool, eg. if different users of the
 243 * pool are operating via different portals.
 244 */
 245struct bman_pool {
 246        /* index of the buffer pool to encapsulate (0-63) */
 247        u32 bpid;
 248        /* Used for hash-table admin when using depletion notifications. */
 249        struct bman_portal *portal;
 250        struct bman_pool *next;
 251};
 252
 253static u32 poll_portal_slow(struct bman_portal *p, u32 is);
 254
 255static irqreturn_t portal_isr(int irq, void *ptr)
 256{
 257        struct bman_portal *p = ptr;
 258        struct bm_portal *portal = &p->p;
 259        u32 clear = p->irq_sources;
 260        u32 is = bm_in(portal, BM_REG_ISR) & p->irq_sources;
 261
 262        if (unlikely(!is))
 263                return IRQ_NONE;
 264
 265        clear |= poll_portal_slow(p, is);
 266        bm_out(portal, BM_REG_ISR, clear);
 267        return IRQ_HANDLED;
 268}
 269
 270/* --- RCR API --- */
 271
 272#define RCR_SHIFT       ilog2(sizeof(struct bm_rcr_entry))
 273#define RCR_CARRY       (uintptr_t)(BM_RCR_SIZE << RCR_SHIFT)
 274
 275/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
 276static struct bm_rcr_entry *rcr_carryclear(struct bm_rcr_entry *p)
 277{
 278        uintptr_t addr = (uintptr_t)p;
 279
 280        addr &= ~RCR_CARRY;
 281
 282        return (struct bm_rcr_entry *)addr;
 283}
 284
 285#ifdef CONFIG_FSL_DPAA_CHECKING
 286/* Bit-wise logic to convert a ring pointer to a ring index */
 287static int rcr_ptr2idx(struct bm_rcr_entry *e)
 288{
 289        return ((uintptr_t)e >> RCR_SHIFT) & (BM_RCR_SIZE - 1);
 290}
 291#endif
 292
 293/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
 294static inline void rcr_inc(struct bm_rcr *rcr)
 295{
 296        /* increment to the next RCR pointer and handle overflow and 'vbit' */
 297        struct bm_rcr_entry *partial = rcr->cursor + 1;
 298
 299        rcr->cursor = rcr_carryclear(partial);
 300        if (partial != rcr->cursor)
 301                rcr->vbit ^= BM_RCR_VERB_VBIT;
 302}
 303
 304static int bm_rcr_get_avail(struct bm_portal *portal)
 305{
 306        struct bm_rcr *rcr = &portal->rcr;
 307
 308        return rcr->available;
 309}
 310
 311static int bm_rcr_get_fill(struct bm_portal *portal)
 312{
 313        struct bm_rcr *rcr = &portal->rcr;
 314
 315        return BM_RCR_SIZE - 1 - rcr->available;
 316}
 317
 318static void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
 319{
 320        struct bm_rcr *rcr = &portal->rcr;
 321
 322        rcr->ithresh = ithresh;
 323        bm_out(portal, BM_REG_RCR_ITR, ithresh);
 324}
 325
 326static void bm_rcr_cce_prefetch(struct bm_portal *portal)
 327{
 328        __maybe_unused struct bm_rcr *rcr = &portal->rcr;
 329
 330        DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
 331        bm_cl_touch_ro(portal, BM_CL_RCR_CI_CENA);
 332}
 333
 334static u8 bm_rcr_cce_update(struct bm_portal *portal)
 335{
 336        struct bm_rcr *rcr = &portal->rcr;
 337        u8 diff, old_ci = rcr->ci;
 338
 339        DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
 340        rcr->ci = bm_ce_in(portal, BM_CL_RCR_CI_CENA) & (BM_RCR_SIZE - 1);
 341        bm_cl_invalidate(portal, BM_CL_RCR_CI_CENA);
 342        diff = dpaa_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
 343        rcr->available += diff;
 344        return diff;
 345}
 346
 347static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
 348{
 349        struct bm_rcr *rcr = &portal->rcr;
 350
 351        DPAA_ASSERT(!rcr->busy);
 352        if (!rcr->available)
 353                return NULL;
 354#ifdef CONFIG_FSL_DPAA_CHECKING
 355        rcr->busy = 1;
 356#endif
 357        dpaa_zero(rcr->cursor);
 358        return rcr->cursor;
 359}
 360
 361static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
 362{
 363        struct bm_rcr *rcr = &portal->rcr;
 364        struct bm_rcr_entry *rcursor;
 365
 366        DPAA_ASSERT(rcr->busy);
 367        DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
 368        DPAA_ASSERT(rcr->available >= 1);
 369        dma_wmb();
 370        rcursor = rcr->cursor;
 371        rcursor->_ncw_verb = myverb | rcr->vbit;
 372        dpaa_flush(rcursor);
 373        rcr_inc(rcr);
 374        rcr->available--;
 375#ifdef CONFIG_FSL_DPAA_CHECKING
 376        rcr->busy = 0;
 377#endif
 378}
 379
 380static int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
 381                       enum bm_rcr_cmode cmode)
 382{
 383        struct bm_rcr *rcr = &portal->rcr;
 384        u32 cfg;
 385        u8 pi;
 386
 387        rcr->ring = portal->addr.ce + BM_CL_RCR;
 388        rcr->ci = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 389        pi = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
 390        rcr->cursor = rcr->ring + pi;
 391        rcr->vbit = (bm_in(portal, BM_REG_RCR_PI_CINH) & BM_RCR_SIZE) ?
 392                BM_RCR_VERB_VBIT : 0;
 393        rcr->available = BM_RCR_SIZE - 1
 394                - dpaa_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
 395        rcr->ithresh = bm_in(portal, BM_REG_RCR_ITR);
 396#ifdef CONFIG_FSL_DPAA_CHECKING
 397        rcr->busy = 0;
 398        rcr->pmode = pmode;
 399        rcr->cmode = cmode;
 400#endif
 401        cfg = (bm_in(portal, BM_REG_CFG) & 0xffffffe0)
 402                | (pmode & 0x3); /* BCSP_CFG::RPM */
 403        bm_out(portal, BM_REG_CFG, cfg);
 404        return 0;
 405}
 406
 407static void bm_rcr_finish(struct bm_portal *portal)
 408{
 409#ifdef CONFIG_FSL_DPAA_CHECKING
 410        struct bm_rcr *rcr = &portal->rcr;
 411        int i;
 412
 413        DPAA_ASSERT(!rcr->busy);
 414
 415        i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
 416        if (i != rcr_ptr2idx(rcr->cursor))
 417                pr_crit("losing uncommitted RCR entries\n");
 418
 419        i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
 420        if (i != rcr->ci)
 421                pr_crit("missing existing RCR completions\n");
 422        if (rcr->ci != rcr_ptr2idx(rcr->cursor))
 423                pr_crit("RCR destroyed unquiesced\n");
 424#endif
 425}
 426
 427/* --- Management command API --- */
 428static int bm_mc_init(struct bm_portal *portal)
 429{
 430        struct bm_mc *mc = &portal->mc;
 431
 432        mc->cr = portal->addr.ce + BM_CL_CR;
 433        mc->rr = portal->addr.ce + BM_CL_RR0;
 434        mc->rridx = (mc->cr->_ncw_verb & BM_MCC_VERB_VBIT) ?
 435                    0 : 1;
 436        mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
 437#ifdef CONFIG_FSL_DPAA_CHECKING
 438        mc->state = mc_idle;
 439#endif
 440        return 0;
 441}
 442
 443static void bm_mc_finish(struct bm_portal *portal)
 444{
 445#ifdef CONFIG_FSL_DPAA_CHECKING
 446        struct bm_mc *mc = &portal->mc;
 447
 448        DPAA_ASSERT(mc->state == mc_idle);
 449        if (mc->state != mc_idle)
 450                pr_crit("Losing incomplete MC command\n");
 451#endif
 452}
 453
 454static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
 455{
 456        struct bm_mc *mc = &portal->mc;
 457
 458        DPAA_ASSERT(mc->state == mc_idle);
 459#ifdef CONFIG_FSL_DPAA_CHECKING
 460        mc->state = mc_user;
 461#endif
 462        dpaa_zero(mc->cr);
 463        return mc->cr;
 464}
 465
 466static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
 467{
 468        struct bm_mc *mc = &portal->mc;
 469        union bm_mc_result *rr = mc->rr + mc->rridx;
 470
 471        DPAA_ASSERT(mc->state == mc_user);
 472        dma_wmb();
 473        mc->cr->_ncw_verb = myverb | mc->vbit;
 474        dpaa_flush(mc->cr);
 475        dpaa_invalidate_touch_ro(rr);
 476#ifdef CONFIG_FSL_DPAA_CHECKING
 477        mc->state = mc_hw;
 478#endif
 479}
 480
 481static inline union bm_mc_result *bm_mc_result(struct bm_portal *portal)
 482{
 483        struct bm_mc *mc = &portal->mc;
 484        union bm_mc_result *rr = mc->rr + mc->rridx;
 485
 486        DPAA_ASSERT(mc->state == mc_hw);
 487        /*
 488         * The inactive response register's verb byte always returns zero until
 489         * its command is submitted and completed. This includes the valid-bit,
 490         * in case you were wondering...
 491         */
 492        if (!rr->verb) {
 493                dpaa_invalidate_touch_ro(rr);
 494                return NULL;
 495        }
 496        mc->rridx ^= 1;
 497        mc->vbit ^= BM_MCC_VERB_VBIT;
 498#ifdef CONFIG_FSL_DPAA_CHECKING
 499        mc->state = mc_idle;
 500#endif
 501        return rr;
 502}
 503
 504static inline int bm_mc_result_timeout(struct bm_portal *portal,
 505                                       union bm_mc_result **mcr)
 506{
 507        int timeout = BM_MCR_TIMEOUT;
 508
 509        do {
 510                *mcr = bm_mc_result(portal);
 511                if (*mcr)
 512                        break;
 513                udelay(1);
 514        } while (--timeout);
 515
 516        return timeout;
 517}
 518
 519/* Disable all BSCN interrupts for the portal */
 520static void bm_isr_bscn_disable(struct bm_portal *portal)
 521{
 522        bm_out(portal, BM_REG_SCN(0), 0);
 523        bm_out(portal, BM_REG_SCN(1), 0);
 524}
 525
 526static int bman_create_portal(struct bman_portal *portal,
 527                              const struct bm_portal_config *c)
 528{
 529        struct bm_portal *p;
 530        int ret;
 531
 532        p = &portal->p;
 533        /*
 534         * prep the low-level portal struct with the mapped addresses from the
 535         * config, everything that follows depends on it and "config" is more
 536         * for (de)reference...
 537         */
 538        p->addr.ce = c->addr_virt_ce;
 539        p->addr.ce_be = c->addr_virt_ce;
 540        p->addr.ci = c->addr_virt_ci;
 541        if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) {
 542                dev_err(c->dev, "RCR initialisation failed\n");
 543                goto fail_rcr;
 544        }
 545        if (bm_mc_init(p)) {
 546                dev_err(c->dev, "MC initialisation failed\n");
 547                goto fail_mc;
 548        }
 549        /*
 550         * Default to all BPIDs disabled, we enable as required at
 551         * run-time.
 552         */
 553        bm_isr_bscn_disable(p);
 554
 555        /* Write-to-clear any stale interrupt status bits */
 556        bm_out(p, BM_REG_ISDR, 0xffffffff);
 557        portal->irq_sources = 0;
 558        bm_out(p, BM_REG_IER, 0);
 559        bm_out(p, BM_REG_ISR, 0xffffffff);
 560        snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
 561        if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) {
 562                dev_err(c->dev, "request_irq() failed\n");
 563                goto fail_irq;
 564        }
 565
 566        if (dpaa_set_portal_irq_affinity(c->dev, c->irq, c->cpu))
 567                goto fail_affinity;
 568
 569        /* Need RCR to be empty before continuing */
 570        ret = bm_rcr_get_fill(p);
 571        if (ret) {
 572                dev_err(c->dev, "RCR unclean\n");
 573                goto fail_rcr_empty;
 574        }
 575        /* Success */
 576        portal->config = c;
 577
 578        bm_out(p, BM_REG_ISDR, 0);
 579        bm_out(p, BM_REG_IIR, 0);
 580
 581        return 0;
 582
 583fail_rcr_empty:
 584fail_affinity:
 585        free_irq(c->irq, portal);
 586fail_irq:
 587        bm_mc_finish(p);
 588fail_mc:
 589        bm_rcr_finish(p);
 590fail_rcr:
 591        return -EIO;
 592}
 593
 594struct bman_portal *bman_create_affine_portal(const struct bm_portal_config *c)
 595{
 596        struct bman_portal *portal;
 597        int err;
 598
 599        portal = &per_cpu(bman_affine_portal, c->cpu);
 600        err = bman_create_portal(portal, c);
 601        if (err)
 602                return NULL;
 603
 604        spin_lock(&affine_mask_lock);
 605        cpumask_set_cpu(c->cpu, &affine_mask);
 606        spin_unlock(&affine_mask_lock);
 607
 608        return portal;
 609}
 610
 611static u32 poll_portal_slow(struct bman_portal *p, u32 is)
 612{
 613        u32 ret = is;
 614
 615        if (is & BM_PIRQ_RCRI) {
 616                bm_rcr_cce_update(&p->p);
 617                bm_rcr_set_ithresh(&p->p, 0);
 618                bm_out(&p->p, BM_REG_ISR, BM_PIRQ_RCRI);
 619                is &= ~BM_PIRQ_RCRI;
 620        }
 621
 622        /* There should be no status register bits left undefined */
 623        DPAA_ASSERT(!is);
 624        return ret;
 625}
 626
 627int bman_p_irqsource_add(struct bman_portal *p, u32 bits)
 628{
 629        unsigned long irqflags;
 630
 631        local_irq_save(irqflags);
 632        p->irq_sources |= bits & BM_PIRQ_VISIBLE;
 633        bm_out(&p->p, BM_REG_IER, p->irq_sources);
 634        local_irq_restore(irqflags);
 635        return 0;
 636}
 637
 638int bm_shutdown_pool(u32 bpid)
 639{
 640        int err = 0;
 641        struct bm_mc_command *bm_cmd;
 642        union bm_mc_result *bm_res;
 643
 644
 645        struct bman_portal *p = get_affine_portal();
 646        while (1) {
 647                /* Acquire buffers until empty */
 648                bm_cmd = bm_mc_start(&p->p);
 649                bm_cmd->bpid = bpid;
 650                bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | 1);
 651                if (!bm_mc_result_timeout(&p->p, &bm_res)) {
 652                        pr_crit("BMan Acquire Command timedout\n");
 653                        err = -ETIMEDOUT;
 654                        goto done;
 655                }
 656                if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
 657                        /* Pool is empty */
 658                        goto done;
 659                }
 660        }
 661done:
 662        put_affine_portal();
 663        return err;
 664}
 665
 666struct gen_pool *bm_bpalloc;
 667
 668static int bm_alloc_bpid_range(u32 *result, u32 count)
 669{
 670        unsigned long addr;
 671
 672        addr = gen_pool_alloc(bm_bpalloc, count);
 673        if (!addr)
 674                return -ENOMEM;
 675
 676        *result = addr & ~DPAA_GENALLOC_OFF;
 677
 678        return 0;
 679}
 680
 681static int bm_release_bpid(u32 bpid)
 682{
 683        int ret;
 684
 685        ret = bm_shutdown_pool(bpid);
 686        if (ret) {
 687                pr_debug("BPID %d leaked\n", bpid);
 688                return ret;
 689        }
 690
 691        gen_pool_free(bm_bpalloc, bpid | DPAA_GENALLOC_OFF, 1);
 692        return 0;
 693}
 694
 695struct bman_pool *bman_new_pool(void)
 696{
 697        struct bman_pool *pool = NULL;
 698        u32 bpid;
 699
 700        if (bm_alloc_bpid_range(&bpid, 1))
 701                return NULL;
 702
 703        pool = kmalloc(sizeof(*pool), GFP_KERNEL);
 704        if (!pool)
 705                goto err;
 706
 707        pool->bpid = bpid;
 708
 709        return pool;
 710err:
 711        bm_release_bpid(bpid);
 712        kfree(pool);
 713        return NULL;
 714}
 715EXPORT_SYMBOL(bman_new_pool);
 716
 717void bman_free_pool(struct bman_pool *pool)
 718{
 719        bm_release_bpid(pool->bpid);
 720
 721        kfree(pool);
 722}
 723EXPORT_SYMBOL(bman_free_pool);
 724
 725int bman_get_bpid(const struct bman_pool *pool)
 726{
 727        return pool->bpid;
 728}
 729EXPORT_SYMBOL(bman_get_bpid);
 730
 731static void update_rcr_ci(struct bman_portal *p, int avail)
 732{
 733        if (avail)
 734                bm_rcr_cce_prefetch(&p->p);
 735        else
 736                bm_rcr_cce_update(&p->p);
 737}
 738
 739int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num)
 740{
 741        struct bman_portal *p;
 742        struct bm_rcr_entry *r;
 743        unsigned long irqflags;
 744        int avail, timeout = 1000; /* 1ms */
 745        int i = num - 1;
 746
 747        DPAA_ASSERT(num > 0 && num <= 8);
 748
 749        do {
 750                p = get_affine_portal();
 751                local_irq_save(irqflags);
 752                avail = bm_rcr_get_avail(&p->p);
 753                if (avail < 2)
 754                        update_rcr_ci(p, avail);
 755                r = bm_rcr_start(&p->p);
 756                local_irq_restore(irqflags);
 757                put_affine_portal();
 758                if (likely(r))
 759                        break;
 760
 761                udelay(1);
 762        } while (--timeout);
 763
 764        if (unlikely(!timeout))
 765                return -ETIMEDOUT;
 766
 767        p = get_affine_portal();
 768        local_irq_save(irqflags);
 769        /*
 770         * we can copy all but the first entry, as this can trigger badness
 771         * with the valid-bit
 772         */
 773        bm_buffer_set64(r->bufs, bm_buffer_get64(bufs));
 774        bm_buffer_set_bpid(r->bufs, pool->bpid);
 775        if (i)
 776                memcpy(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));
 777
 778        bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
 779                          (num & BM_RCR_VERB_BUFCOUNT_MASK));
 780
 781        local_irq_restore(irqflags);
 782        put_affine_portal();
 783        return 0;
 784}
 785EXPORT_SYMBOL(bman_release);
 786
 787int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num)
 788{
 789        struct bman_portal *p = get_affine_portal();
 790        struct bm_mc_command *mcc;
 791        union bm_mc_result *mcr;
 792        int ret;
 793
 794        DPAA_ASSERT(num > 0 && num <= 8);
 795
 796        mcc = bm_mc_start(&p->p);
 797        mcc->bpid = pool->bpid;
 798        bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
 799                     (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
 800        if (!bm_mc_result_timeout(&p->p, &mcr)) {
 801                put_affine_portal();
 802                pr_crit("BMan Acquire Timeout\n");
 803                return -ETIMEDOUT;
 804        }
 805        ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
 806        if (bufs)
 807                memcpy(&bufs[0], &mcr->bufs[0], num * sizeof(bufs[0]));
 808
 809        put_affine_portal();
 810        if (ret != num)
 811                ret = -ENOMEM;
 812        return ret;
 813}
 814EXPORT_SYMBOL(bman_acquire);
 815
 816const struct bm_portal_config *
 817bman_get_bm_portal_config(const struct bman_portal *portal)
 818{
 819        return portal->config;
 820}
 821