linux/drivers/scsi/csiostor/csio_wr.c
<<
>>
Prefs
   1/*
   2 * This file is part of the Chelsio FCoE driver for Linux.
   3 *
   4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 */
  34
  35#include <linux/kernel.h>
  36#include <linux/string.h>
  37#include <linux/compiler.h>
  38#include <linux/slab.h>
  39#include <asm/page.h>
  40#include <linux/cache.h>
  41
  42#include "csio_hw.h"
  43#include "csio_wr.h"
  44#include "csio_mb.h"
  45#include "csio_defs.h"
  46
  47int csio_intr_coalesce_cnt;             /* value:SGE_INGRESS_RX_THRESHOLD[0] */
  48static int csio_sge_thresh_reg;         /* SGE_INGRESS_RX_THRESHOLD[0] */
  49
  50int csio_intr_coalesce_time = 10;       /* value:SGE_TIMER_VALUE_1 */
  51static int csio_sge_timer_reg = 1;
  52
  53#define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val)                            \
  54        csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg)
  55
  56static void
  57csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)
  58{
  59        sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0 +
  60                                                        reg * sizeof(uint32_t));
  61}
  62
  63/* Free list buffer size */
  64static inline uint32_t
  65csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)
  66{
  67        return sge->sge_fl_buf_size[buf->paddr & 0xF];
  68}
  69
  70/* Size of the egress queue status page */
  71static inline uint32_t
  72csio_wr_qstat_pgsz(struct csio_hw *hw)
  73{
  74        return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE(1)) ?  128 : 64;
  75}
  76
  77/* Ring freelist doorbell */
  78static inline void
  79csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)
  80{
  81        /*
  82         * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ
  83         * number of bytes in the freelist queue. This translates to atleast
  84         * 8 freelist buffer pointers (since each pointer is 8 bytes).
  85         */
  86        if (flq->inc_idx >= 8) {
  87                csio_wr_reg32(hw, DBPRIO(1) | QID(flq->un.fl.flid) |
  88                              PIDX(flq->inc_idx / 8),
  89                              MYPF_REG(SGE_PF_KDOORBELL));
  90                flq->inc_idx &= 7;
  91        }
  92}
  93
  94/* Write a 0 cidx increment value to enable SGE interrupts for this queue */
  95static void
  96csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)
  97{
  98        csio_wr_reg32(hw, CIDXINC(0)            |
  99                          INGRESSQID(iqid)      |
 100                          TIMERREG(X_TIMERREG_RESTART_COUNTER),
 101                          MYPF_REG(SGE_PF_GTS));
 102}
 103
 104/*
 105 * csio_wr_fill_fl - Populate the FL buffers of a FL queue.
 106 * @hw: HW module.
 107 * @flq: Freelist queue.
 108 *
 109 * Fill up freelist buffer entries with buffers of size specified
 110 * in the size register.
 111 *
 112 */
 113static int
 114csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)
 115{
 116        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 117        struct csio_sge *sge = &wrm->sge;
 118        __be64 *d = (__be64 *)(flq->vstart);
 119        struct csio_dma_buf *buf = &flq->un.fl.bufs[0];
 120        uint64_t paddr;
 121        int sreg = flq->un.fl.sreg;
 122        int n = flq->credits;
 123
 124        while (n--) {
 125                buf->len = sge->sge_fl_buf_size[sreg];
 126                buf->vaddr = pci_alloc_consistent(hw->pdev, buf->len,
 127                                                  &buf->paddr);
 128                if (!buf->vaddr) {
 129                        csio_err(hw, "Could only fill %d buffers!\n", n + 1);
 130                        return -ENOMEM;
 131                }
 132
 133                paddr = buf->paddr | (sreg & 0xF);
 134
 135                *d++ = cpu_to_be64(paddr);
 136                buf++;
 137        }
 138
 139        return 0;
 140}
 141
 142/*
 143 * csio_wr_update_fl -
 144 * @hw: HW module.
 145 * @flq: Freelist queue.
 146 *
 147 *
 148 */
 149static inline void
 150csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)
 151{
 152
 153        flq->inc_idx += n;
 154        flq->pidx += n;
 155        if (unlikely(flq->pidx >= flq->credits))
 156                flq->pidx -= (uint16_t)flq->credits;
 157
 158        CSIO_INC_STATS(flq, n_flq_refill);
 159}
 160
 161/*
 162 * csio_wr_alloc_q - Allocate a WR queue and initialize it.
 163 * @hw: HW module
 164 * @qsize: Size of the queue in bytes
 165 * @wrsize: Since of WR in this queue, if fixed.
 166 * @type: Type of queue (Ingress/Egress/Freelist)
 167 * @owner: Module that owns this queue.
 168 * @nflb: Number of freelist buffers for FL.
 169 * @sreg: What is the FL buffer size register?
 170 * @iq_int_handler: Ingress queue handler in INTx mode.
 171 *
 172 * This function allocates and sets up a queue for the caller
 173 * of size qsize, aligned at the required boundary. This is subject to
 174 * be free entries being available in the queue array. If one is found,
 175 * it is initialized with the allocated queue, marked as being used (owner),
 176 * and a handle returned to the caller in form of the queue's index
 177 * into the q_arr array.
 178 * If user has indicated a freelist (by specifying nflb > 0), create
 179 * another queue (with its own index into q_arr) for the freelist. Allocate
 180 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist
 181 * idx in the ingress queue's flq.idx. This is how a Freelist is associated
 182 * with its owning ingress queue.
 183 */
 184int
 185csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,
 186                uint16_t type, void *owner, uint32_t nflb, int sreg,
 187                iq_handler_t iq_intx_handler)
 188{
 189        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 190        struct csio_q   *q, *flq;
 191        int             free_idx = wrm->free_qidx;
 192        int             ret_idx = free_idx;
 193        uint32_t        qsz;
 194        int flq_idx;
 195
 196        if (free_idx >= wrm->num_q) {
 197                csio_err(hw, "No more free queues.\n");
 198                return -1;
 199        }
 200
 201        switch (type) {
 202        case CSIO_EGRESS:
 203                qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);
 204                break;
 205        case CSIO_INGRESS:
 206                switch (wrsize) {
 207                case 16:
 208                case 32:
 209                case 64:
 210                case 128:
 211                        break;
 212                default:
 213                        csio_err(hw, "Invalid Ingress queue WR size:%d\n",
 214                                    wrsize);
 215                        return -1;
 216                }
 217
 218                /*
 219                 * Number of elements must be a multiple of 16
 220                 * So this includes status page size
 221                 */
 222                qsz = ALIGN(qsize/wrsize, 16) * wrsize;
 223
 224                break;
 225        case CSIO_FREELIST:
 226                qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);
 227                break;
 228        default:
 229                csio_err(hw, "Invalid queue type: 0x%x\n", type);
 230                return -1;
 231        }
 232
 233        q = wrm->q_arr[free_idx];
 234
 235        q->vstart = pci_alloc_consistent(hw->pdev, qsz, &q->pstart);
 236        if (!q->vstart) {
 237                csio_err(hw,
 238                         "Failed to allocate DMA memory for "
 239                         "queue at id: %d size: %d\n", free_idx, qsize);
 240                return -1;
 241        }
 242
 243        /*
 244         * We need to zero out the contents, importantly for ingress,
 245         * since we start with a generatiom bit of 1 for ingress.
 246         */
 247        memset(q->vstart, 0, qsz);
 248
 249        q->type         = type;
 250        q->owner        = owner;
 251        q->pidx         = q->cidx = q->inc_idx = 0;
 252        q->size         = qsz;
 253        q->wr_sz        = wrsize;       /* If using fixed size WRs */
 254
 255        wrm->free_qidx++;
 256
 257        if (type == CSIO_INGRESS) {
 258                /* Since queue area is set to zero */
 259                q->un.iq.genbit = 1;
 260
 261                /*
 262                 * Ingress queue status page size is always the size of
 263                 * the ingress queue entry.
 264                 */
 265                q->credits      = (qsz - q->wr_sz) / q->wr_sz;
 266                q->vwrap        = (void *)((uintptr_t)(q->vstart) + qsz
 267                                                        - q->wr_sz);
 268
 269                /* Allocate memory for FL if requested */
 270                if (nflb > 0) {
 271                        flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),
 272                                                  sizeof(__be64), CSIO_FREELIST,
 273                                                  owner, 0, sreg, NULL);
 274                        if (flq_idx == -1) {
 275                                csio_err(hw,
 276                                         "Failed to allocate FL queue"
 277                                         " for IQ idx:%d\n", free_idx);
 278                                return -1;
 279                        }
 280
 281                        /* Associate the new FL with the Ingress quue */
 282                        q->un.iq.flq_idx = flq_idx;
 283
 284                        flq = wrm->q_arr[q->un.iq.flq_idx];
 285                        flq->un.fl.bufs = kzalloc(flq->credits *
 286                                                  sizeof(struct csio_dma_buf),
 287                                                  GFP_KERNEL);
 288                        if (!flq->un.fl.bufs) {
 289                                csio_err(hw,
 290                                         "Failed to allocate FL queue bufs"
 291                                         " for IQ idx:%d\n", free_idx);
 292                                return -1;
 293                        }
 294
 295                        flq->un.fl.packen = 0;
 296                        flq->un.fl.offset = 0;
 297                        flq->un.fl.sreg = sreg;
 298
 299                        /* Fill up the free list buffers */
 300                        if (csio_wr_fill_fl(hw, flq))
 301                                return -1;
 302
 303                        /*
 304                         * Make sure in a FLQ, atleast 1 credit (8 FL buffers)
 305                         * remains unpopulated,otherwise HW thinks
 306                         * FLQ is empty.
 307                         */
 308                        flq->pidx = flq->inc_idx = flq->credits - 8;
 309                } else {
 310                        q->un.iq.flq_idx = -1;
 311                }
 312
 313                /* Associate the IQ INTx handler. */
 314                q->un.iq.iq_intx_handler = iq_intx_handler;
 315
 316                csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;
 317
 318        } else if (type == CSIO_EGRESS) {
 319                q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;
 320                q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
 321                                                - csio_wr_qstat_pgsz(hw));
 322                csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;
 323        } else { /* Freelist */
 324                q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);
 325                q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
 326                                                - csio_wr_qstat_pgsz(hw));
 327                csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;
 328        }
 329
 330        return ret_idx;
 331}
 332
 333/*
 334 * csio_wr_iq_create_rsp - Response handler for IQ creation.
 335 * @hw: The HW module.
 336 * @mbp: Mailbox.
 337 * @iq_idx: Ingress queue that got created.
 338 *
 339 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.
 340 */
 341static int
 342csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
 343{
 344        struct csio_iq_params iqp;
 345        enum fw_retval retval;
 346        uint32_t iq_id;
 347        int flq_idx;
 348
 349        memset(&iqp, 0, sizeof(struct csio_iq_params));
 350
 351        csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);
 352
 353        if (retval != FW_SUCCESS) {
 354                csio_err(hw, "IQ cmd returned 0x%x!\n", retval);
 355                mempool_free(mbp, hw->mb_mempool);
 356                return -EINVAL;
 357        }
 358
 359        csio_q_iqid(hw, iq_idx)         = iqp.iqid;
 360        csio_q_physiqid(hw, iq_idx)     = iqp.physiqid;
 361        csio_q_pidx(hw, iq_idx)         = csio_q_cidx(hw, iq_idx) = 0;
 362        csio_q_inc_idx(hw, iq_idx)      = 0;
 363
 364        /* Actual iq-id. */
 365        iq_id = iqp.iqid - hw->wrm.fw_iq_start;
 366
 367        /* Set the iq-id to iq map table. */
 368        if (iq_id >= CSIO_MAX_IQ) {
 369                csio_err(hw,
 370                         "Exceeding MAX_IQ(%d) supported!"
 371                         " iqid:%d rel_iqid:%d FW iq_start:%d\n",
 372                         CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);
 373                mempool_free(mbp, hw->mb_mempool);
 374                return -EINVAL;
 375        }
 376        csio_q_set_intr_map(hw, iq_idx, iq_id);
 377
 378        /*
 379         * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE
 380         * ingress context of this queue. This will block interrupts to
 381         * this queue until the next GTS write. Therefore, we do a
 382         * 0-cidx increment GTS write for this queue just to clear the
 383         * interrupt_sent bit. This will re-enable interrupts to this
 384         * queue.
 385         */
 386        csio_wr_sge_intr_enable(hw, iqp.physiqid);
 387
 388        flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
 389        if (flq_idx != -1) {
 390                struct csio_q *flq = hw->wrm.q_arr[flq_idx];
 391
 392                csio_q_flid(hw, flq_idx) = iqp.fl0id;
 393                csio_q_cidx(hw, flq_idx) = 0;
 394                csio_q_pidx(hw, flq_idx)    = csio_q_credits(hw, flq_idx) - 8;
 395                csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
 396
 397                /* Now update SGE about the buffers allocated during init */
 398                csio_wr_ring_fldb(hw, flq);
 399        }
 400
 401        mempool_free(mbp, hw->mb_mempool);
 402
 403        return 0;
 404}
 405
 406/*
 407 * csio_wr_iq_create - Configure an Ingress queue with FW.
 408 * @hw: The HW module.
 409 * @priv: Private data object.
 410 * @iq_idx: Ingress queue index in the WR module.
 411 * @vec: MSIX vector.
 412 * @portid: PCIE Channel to be associated with this queue.
 413 * @async: Is this a FW asynchronous message handling queue?
 414 * @cbfn: Completion callback.
 415 *
 416 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox
 417 * with alloc/write bits set.
 418 */
 419int
 420csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,
 421                  uint32_t vec, uint8_t portid, bool async,
 422                  void (*cbfn) (struct csio_hw *, struct csio_mb *))
 423{
 424        struct csio_mb  *mbp;
 425        struct csio_iq_params iqp;
 426        int flq_idx;
 427
 428        memset(&iqp, 0, sizeof(struct csio_iq_params));
 429        csio_q_portid(hw, iq_idx) = portid;
 430
 431        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 432        if (!mbp) {
 433                csio_err(hw, "IQ command out of memory!\n");
 434                return -ENOMEM;
 435        }
 436
 437        switch (hw->intr_mode) {
 438        case CSIO_IM_INTX:
 439        case CSIO_IM_MSI:
 440                /* For interrupt forwarding queue only */
 441                if (hw->intr_iq_idx == iq_idx)
 442                        iqp.iqandst     = X_INTERRUPTDESTINATION_PCIE;
 443                else
 444                        iqp.iqandst     = X_INTERRUPTDESTINATION_IQ;
 445                iqp.iqandstindex        =
 446                        csio_q_physiqid(hw, hw->intr_iq_idx);
 447                break;
 448        case CSIO_IM_MSIX:
 449                iqp.iqandst             = X_INTERRUPTDESTINATION_PCIE;
 450                iqp.iqandstindex        = (uint16_t)vec;
 451                break;
 452        case CSIO_IM_NONE:
 453                mempool_free(mbp, hw->mb_mempool);
 454                return -EINVAL;
 455        }
 456
 457        /* Pass in the ingress queue cmd parameters */
 458        iqp.pfn                 = hw->pfn;
 459        iqp.vfn                 = 0;
 460        iqp.iq_start            = 1;
 461        iqp.viid                = 0;
 462        iqp.type                = FW_IQ_TYPE_FL_INT_CAP;
 463        iqp.iqasynch            = async;
 464        if (csio_intr_coalesce_cnt)
 465                iqp.iqanus      = X_UPDATESCHEDULING_COUNTER_OPTTIMER;
 466        else
 467                iqp.iqanus      = X_UPDATESCHEDULING_TIMER;
 468        iqp.iqanud              = X_UPDATEDELIVERY_INTERRUPT;
 469        iqp.iqpciech            = portid;
 470        iqp.iqintcntthresh      = (uint8_t)csio_sge_thresh_reg;
 471
 472        switch (csio_q_wr_sz(hw, iq_idx)) {
 473        case 16:
 474                iqp.iqesize = 0; break;
 475        case 32:
 476                iqp.iqesize = 1; break;
 477        case 64:
 478                iqp.iqesize = 2; break;
 479        case 128:
 480                iqp.iqesize = 3; break;
 481        }
 482
 483        iqp.iqsize              = csio_q_size(hw, iq_idx) /
 484                                                csio_q_wr_sz(hw, iq_idx);
 485        iqp.iqaddr              = csio_q_pstart(hw, iq_idx);
 486
 487        flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
 488        if (flq_idx != -1) {
 489                struct csio_q *flq = hw->wrm.q_arr[flq_idx];
 490
 491                iqp.fl0paden    = 1;
 492                iqp.fl0packen   = flq->un.fl.packen ? 1 : 0;
 493                iqp.fl0fbmin    = X_FETCHBURSTMIN_64B;
 494                iqp.fl0fbmax    = X_FETCHBURSTMAX_512B;
 495                iqp.fl0size     = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;
 496                iqp.fl0addr     = csio_q_pstart(hw, flq_idx);
 497        }
 498
 499        csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
 500
 501        if (csio_mb_issue(hw, mbp)) {
 502                csio_err(hw, "Issue of IQ cmd failed!\n");
 503                mempool_free(mbp, hw->mb_mempool);
 504                return -EINVAL;
 505        }
 506
 507        if (cbfn != NULL)
 508                return 0;
 509
 510        return csio_wr_iq_create_rsp(hw, mbp, iq_idx);
 511}
 512
 513/*
 514 * csio_wr_eq_create_rsp - Response handler for EQ creation.
 515 * @hw: The HW module.
 516 * @mbp: Mailbox.
 517 * @eq_idx: Egress queue that got created.
 518 *
 519 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.
 520 */
 521static int
 522csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
 523{
 524        struct csio_eq_params eqp;
 525        enum fw_retval retval;
 526
 527        memset(&eqp, 0, sizeof(struct csio_eq_params));
 528
 529        csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);
 530
 531        if (retval != FW_SUCCESS) {
 532                csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);
 533                mempool_free(mbp, hw->mb_mempool);
 534                return -EINVAL;
 535        }
 536
 537        csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid;
 538        csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;
 539        csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0;
 540        csio_q_inc_idx(hw, eq_idx) = 0;
 541
 542        mempool_free(mbp, hw->mb_mempool);
 543
 544        return 0;
 545}
 546
 547/*
 548 * csio_wr_eq_create - Configure an Egress queue with FW.
 549 * @hw: HW module.
 550 * @priv: Private data.
 551 * @eq_idx: Egress queue index in the WR module.
 552 * @iq_idx: Associated ingress queue index.
 553 * @cbfn: Completion callback.
 554 *
 555 * This API configures a offload egress queue with FW by issuing a
 556 * FW_EQ_OFLD_CMD  (with alloc + write ) mailbox.
 557 */
 558int
 559csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,
 560                  int iq_idx, uint8_t portid,
 561                  void (*cbfn) (struct csio_hw *, struct csio_mb *))
 562{
 563        struct csio_mb  *mbp;
 564        struct csio_eq_params eqp;
 565
 566        memset(&eqp, 0, sizeof(struct csio_eq_params));
 567
 568        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 569        if (!mbp) {
 570                csio_err(hw, "EQ command out of memory!\n");
 571                return -ENOMEM;
 572        }
 573
 574        eqp.pfn                 = hw->pfn;
 575        eqp.vfn                 = 0;
 576        eqp.eqstart             = 1;
 577        eqp.hostfcmode          = X_HOSTFCMODE_STATUS_PAGE;
 578        eqp.iqid                = csio_q_iqid(hw, iq_idx);
 579        eqp.fbmin               = X_FETCHBURSTMIN_64B;
 580        eqp.fbmax               = X_FETCHBURSTMAX_512B;
 581        eqp.cidxfthresh         = 0;
 582        eqp.pciechn             = portid;
 583        eqp.eqsize              = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;
 584        eqp.eqaddr              = csio_q_pstart(hw, eq_idx);
 585
 586        csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,
 587                                    &eqp, cbfn);
 588
 589        if (csio_mb_issue(hw, mbp)) {
 590                csio_err(hw, "Issue of EQ OFLD cmd failed!\n");
 591                mempool_free(mbp, hw->mb_mempool);
 592                return -EINVAL;
 593        }
 594
 595        if (cbfn != NULL)
 596                return 0;
 597
 598        return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);
 599}
 600
 601/*
 602 * csio_wr_iq_destroy_rsp - Response handler for IQ removal.
 603 * @hw: The HW module.
 604 * @mbp: Mailbox.
 605 * @iq_idx: Ingress queue that was freed.
 606 *
 607 * Handle FW_IQ_CMD (free) mailbox completion.
 608 */
 609static int
 610csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
 611{
 612        enum fw_retval retval = csio_mb_fw_retval(mbp);
 613        int rv = 0;
 614
 615        if (retval != FW_SUCCESS)
 616                rv = -EINVAL;
 617
 618        mempool_free(mbp, hw->mb_mempool);
 619
 620        return rv;
 621}
 622
 623/*
 624 * csio_wr_iq_destroy - Free an ingress queue.
 625 * @hw: The HW module.
 626 * @priv: Private data object.
 627 * @iq_idx: Ingress queue index to destroy
 628 * @cbfn: Completion callback.
 629 *
 630 * This API frees an ingress queue by issuing the FW_IQ_CMD
 631 * with the free bit set.
 632 */
 633static int
 634csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,
 635                   void (*cbfn)(struct csio_hw *, struct csio_mb *))
 636{
 637        int rv = 0;
 638        struct csio_mb  *mbp;
 639        struct csio_iq_params iqp;
 640        int flq_idx;
 641
 642        memset(&iqp, 0, sizeof(struct csio_iq_params));
 643
 644        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 645        if (!mbp)
 646                return -ENOMEM;
 647
 648        iqp.pfn         = hw->pfn;
 649        iqp.vfn         = 0;
 650        iqp.iqid        = csio_q_iqid(hw, iq_idx);
 651        iqp.type        = FW_IQ_TYPE_FL_INT_CAP;
 652
 653        flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
 654        if (flq_idx != -1)
 655                iqp.fl0id = csio_q_flid(hw, flq_idx);
 656        else
 657                iqp.fl0id = 0xFFFF;
 658
 659        iqp.fl1id = 0xFFFF;
 660
 661        csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
 662
 663        rv = csio_mb_issue(hw, mbp);
 664        if (rv != 0) {
 665                mempool_free(mbp, hw->mb_mempool);
 666                return rv;
 667        }
 668
 669        if (cbfn != NULL)
 670                return 0;
 671
 672        return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);
 673}
 674
 675/*
 676 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.
 677 * @hw: The HW module.
 678 * @mbp: Mailbox.
 679 * @eq_idx: Egress queue that was freed.
 680 *
 681 * Handle FW_OFLD_EQ_CMD (free) mailbox completion.
 682 */
 683static int
 684csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
 685{
 686        enum fw_retval retval = csio_mb_fw_retval(mbp);
 687        int rv = 0;
 688
 689        if (retval != FW_SUCCESS)
 690                rv = -EINVAL;
 691
 692        mempool_free(mbp, hw->mb_mempool);
 693
 694        return rv;
 695}
 696
 697/*
 698 * csio_wr_eq_destroy - Free an Egress queue.
 699 * @hw: The HW module.
 700 * @priv: Private data object.
 701 * @eq_idx: Egress queue index to destroy
 702 * @cbfn: Completion callback.
 703 *
 704 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD
 705 * with the free bit set.
 706 */
 707static int
 708csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,
 709                   void (*cbfn) (struct csio_hw *, struct csio_mb *))
 710{
 711        int rv = 0;
 712        struct csio_mb  *mbp;
 713        struct csio_eq_params eqp;
 714
 715        memset(&eqp, 0, sizeof(struct csio_eq_params));
 716
 717        mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 718        if (!mbp)
 719                return -ENOMEM;
 720
 721        eqp.pfn         = hw->pfn;
 722        eqp.vfn         = 0;
 723        eqp.eqid        = csio_q_eqid(hw, eq_idx);
 724
 725        csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);
 726
 727        rv = csio_mb_issue(hw, mbp);
 728        if (rv != 0) {
 729                mempool_free(mbp, hw->mb_mempool);
 730                return rv;
 731        }
 732
 733        if (cbfn != NULL)
 734                return 0;
 735
 736        return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);
 737}
 738
 739/*
 740 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page
 741 * @hw: HW module
 742 * @qidx: Egress queue index
 743 *
 744 * Cleanup the Egress queue status page.
 745 */
 746static void
 747csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)
 748{
 749        struct csio_q   *q = csio_hw_to_wrm(hw)->q_arr[qidx];
 750        struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
 751
 752        memset(stp, 0, sizeof(*stp));
 753}
 754
 755/*
 756 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ
 757 * @hw: HW module
 758 * @qidx: Ingress queue index
 759 *
 760 * Cleanup the footer entries in the given ingress queue,
 761 * set to 1 the internal copy of genbit.
 762 */
 763static void
 764csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)
 765{
 766        struct csio_wrm *wrm    = csio_hw_to_wrm(hw);
 767        struct csio_q   *q      = wrm->q_arr[qidx];
 768        void *wr;
 769        struct csio_iqwr_footer *ftr;
 770        uint32_t i = 0;
 771
 772        /* set to 1 since we are just about zero out genbit */
 773        q->un.iq.genbit = 1;
 774
 775        for (i = 0; i < q->credits; i++) {
 776                /* Get the WR */
 777                wr = (void *)((uintptr_t)q->vstart +
 778                                           (i * q->wr_sz));
 779                /* Get the footer */
 780                ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
 781                                          (q->wr_sz - sizeof(*ftr)));
 782                /* Zero out footer */
 783                memset(ftr, 0, sizeof(*ftr));
 784        }
 785}
 786
 787int
 788csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)
 789{
 790        int i, flq_idx;
 791        struct csio_q *q;
 792        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 793        int rv;
 794
 795        for (i = 0; i < wrm->free_qidx; i++) {
 796                q = wrm->q_arr[i];
 797
 798                switch (q->type) {
 799                case CSIO_EGRESS:
 800                        if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {
 801                                csio_wr_cleanup_eq_stpg(hw, i);
 802                                if (!cmd) {
 803                                        csio_q_eqid(hw, i) = CSIO_MAX_QID;
 804                                        continue;
 805                                }
 806
 807                                rv = csio_wr_eq_destroy(hw, NULL, i, NULL);
 808                                if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
 809                                        cmd = false;
 810
 811                                csio_q_eqid(hw, i) = CSIO_MAX_QID;
 812                        }
 813                case CSIO_INGRESS:
 814                        if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {
 815                                csio_wr_cleanup_iq_ftr(hw, i);
 816                                if (!cmd) {
 817                                        csio_q_iqid(hw, i) = CSIO_MAX_QID;
 818                                        flq_idx = csio_q_iq_flq_idx(hw, i);
 819                                        if (flq_idx != -1)
 820                                                csio_q_flid(hw, flq_idx) =
 821                                                                CSIO_MAX_QID;
 822                                        continue;
 823                                }
 824
 825                                rv = csio_wr_iq_destroy(hw, NULL, i, NULL);
 826                                if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
 827                                        cmd = false;
 828
 829                                csio_q_iqid(hw, i) = CSIO_MAX_QID;
 830                                flq_idx = csio_q_iq_flq_idx(hw, i);
 831                                if (flq_idx != -1)
 832                                        csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;
 833                        }
 834                default:
 835                        break;
 836                }
 837        }
 838
 839        hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;
 840
 841        return 0;
 842}
 843
 844/*
 845 * csio_wr_get - Get requested size of WR entry/entries from queue.
 846 * @hw: HW module.
 847 * @qidx: Index of queue.
 848 * @size: Cumulative size of Work request(s).
 849 * @wrp: Work request pair.
 850 *
 851 * If requested credits are available, return the start address of the
 852 * work request in the work request pair. Set pidx accordingly and
 853 * return.
 854 *
 855 * NOTE about WR pair:
 856 * ==================
 857 * A WR can start towards the end of a queue, and then continue at the
 858 * beginning, since the queue is considered to be circular. This will
 859 * require a pair of address/size to be passed back to the caller -
 860 * hence Work request pair format.
 861 */
 862int
 863csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,
 864            struct csio_wr_pair *wrp)
 865{
 866        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 867        struct csio_q *q = wrm->q_arr[qidx];
 868        void *cwr = (void *)((uintptr_t)(q->vstart) +
 869                                                (q->pidx * CSIO_QCREDIT_SZ));
 870        struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
 871        uint16_t cidx = q->cidx = ntohs(stp->cidx);
 872        uint16_t pidx = q->pidx;
 873        uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ);
 874        int req_credits = req_sz / CSIO_QCREDIT_SZ;
 875        int credits;
 876
 877        CSIO_DB_ASSERT(q->owner != NULL);
 878        CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
 879        CSIO_DB_ASSERT(cidx <= q->credits);
 880
 881        /* Calculate credits */
 882        if (pidx > cidx) {
 883                credits = q->credits - (pidx - cidx) - 1;
 884        } else if (cidx > pidx) {
 885                credits = cidx - pidx - 1;
 886        } else {
 887                /* cidx == pidx, empty queue */
 888                credits = q->credits;
 889                CSIO_INC_STATS(q, n_qempty);
 890        }
 891
 892        /*
 893         * Check if we have enough credits.
 894         * credits = 1 implies queue is full.
 895         */
 896        if (!credits || (req_credits > credits)) {
 897                CSIO_INC_STATS(q, n_qfull);
 898                return -EBUSY;
 899        }
 900
 901        /*
 902         * If we are here, we have enough credits to satisfy the
 903         * request. Check if we are near the end of q, and if WR spills over.
 904         * If it does, use the first addr/size to cover the queue until
 905         * the end. Fit the remainder portion of the request at the top
 906         * of queue and return it in the second addr/len. Set pidx
 907         * accordingly.
 908         */
 909        if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {
 910                wrp->addr1 = cwr;
 911                wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);
 912                wrp->addr2 = q->vstart;
 913                wrp->size2 = req_sz - wrp->size1;
 914                q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /
 915                                                        CSIO_QCREDIT_SZ);
 916                CSIO_INC_STATS(q, n_qwrap);
 917                CSIO_INC_STATS(q, n_eq_wr_split);
 918        } else {
 919                wrp->addr1 = cwr;
 920                wrp->size1 = req_sz;
 921                wrp->addr2 = NULL;
 922                wrp->size2 = 0;
 923                q->pidx += (uint16_t)req_credits;
 924
 925                /* We are the end of queue, roll back pidx to top of queue */
 926                if (unlikely(q->pidx == q->credits)) {
 927                        q->pidx = 0;
 928                        CSIO_INC_STATS(q, n_qwrap);
 929                }
 930        }
 931
 932        q->inc_idx = (uint16_t)req_credits;
 933
 934        CSIO_INC_STATS(q, n_tot_reqs);
 935
 936        return 0;
 937}
 938
 939/*
 940 * csio_wr_copy_to_wrp - Copies given data into WR.
 941 * @data_buf - Data buffer
 942 * @wrp - Work request pair.
 943 * @wr_off - Work request offset.
 944 * @data_len - Data length.
 945 *
 946 * Copies the given data in Work Request. Work request pair(wrp) specifies
 947 * address information of Work request.
 948 * Returns: none
 949 */
 950void
 951csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,
 952                   uint32_t wr_off, uint32_t data_len)
 953{
 954        uint32_t nbytes;
 955
 956        /* Number of space available in buffer addr1 of WRP */
 957        nbytes = ((wrp->size1 - wr_off) >= data_len) ?
 958                                        data_len : (wrp->size1 - wr_off);
 959
 960        memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);
 961        data_len -= nbytes;
 962
 963        /* Write the remaining data from the begining of circular buffer */
 964        if (data_len) {
 965                CSIO_DB_ASSERT(data_len <= wrp->size2);
 966                CSIO_DB_ASSERT(wrp->addr2 != NULL);
 967                memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);
 968        }
 969}
 970
 971/*
 972 * csio_wr_issue - Notify chip of Work request.
 973 * @hw: HW module.
 974 * @qidx: Index of queue.
 975 * @prio: 0: Low priority, 1: High priority
 976 *
 977 * Rings the SGE Doorbell by writing the current producer index of the passed
 978 * in queue into the register.
 979 *
 980 */
 981int
 982csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)
 983{
 984        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 985        struct csio_q *q = wrm->q_arr[qidx];
 986
 987        CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
 988
 989        wmb();
 990        /* Ring SGE Doorbell writing q->pidx into it */
 991        csio_wr_reg32(hw, DBPRIO(prio) | QID(q->un.eq.physeqid) |
 992                      PIDX(q->inc_idx), MYPF_REG(SGE_PF_KDOORBELL));
 993        q->inc_idx = 0;
 994
 995        return 0;
 996}
 997
 998static inline uint32_t
 999csio_wr_avail_qcredits(struct csio_q *q)
1000{
1001        if (q->pidx > q->cidx)
1002                return q->pidx - q->cidx;
1003        else if (q->cidx > q->pidx)
1004                return q->credits - (q->cidx - q->pidx);
1005        else
1006                return 0;       /* cidx == pidx, empty queue */
1007}
1008
1009/*
1010 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.
1011 * @hw: HW module.
1012 * @flq: The freelist queue.
1013 *
1014 * Invalidate the driver's version of a freelist buffer entry,
1015 * without freeing the associated the DMA memory. The entry
1016 * to be invalidated is picked up from the current Free list
1017 * queue cidx.
1018 *
1019 */
1020static inline void
1021csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)
1022{
1023        flq->cidx++;
1024        if (flq->cidx == flq->credits) {
1025                flq->cidx = 0;
1026                CSIO_INC_STATS(flq, n_qwrap);
1027        }
1028}
1029
1030/*
1031 * csio_wr_process_fl - Process a freelist completion.
1032 * @hw: HW module.
1033 * @q: The ingress queue attached to the Freelist.
1034 * @wr: The freelist completion WR in the ingress queue.
1035 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer
1036 * @iq_handler: Caller's handler for this completion.
1037 * @priv: Private pointer of caller
1038 *
1039 */
1040static inline void
1041csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,
1042                   void *wr, uint32_t len_to_qid,
1043                   void (*iq_handler)(struct csio_hw *, void *,
1044                                      uint32_t, struct csio_fl_dma_buf *,
1045                                      void *),
1046                   void *priv)
1047{
1048        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1049        struct csio_sge *sge = &wrm->sge;
1050        struct csio_fl_dma_buf flb;
1051        struct csio_dma_buf *buf, *fbuf;
1052        uint32_t bufsz, len, lastlen = 0;
1053        int n;
1054        struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];
1055
1056        CSIO_DB_ASSERT(flq != NULL);
1057
1058        len = len_to_qid;
1059
1060        if (len & IQWRF_NEWBUF) {
1061                if (flq->un.fl.offset > 0) {
1062                        csio_wr_inval_flq_buf(hw, flq);
1063                        flq->un.fl.offset = 0;
1064                }
1065                len = IQWRF_LEN_GET(len);
1066        }
1067
1068        CSIO_DB_ASSERT(len != 0);
1069
1070        flb.totlen = len;
1071
1072        /* Consume all freelist buffers used for len bytes */
1073        for (n = 0, fbuf = flb.flbufs; ; n++, fbuf++) {
1074                buf = &flq->un.fl.bufs[flq->cidx];
1075                bufsz = csio_wr_fl_bufsz(sge, buf);
1076
1077                fbuf->paddr     = buf->paddr;
1078                fbuf->vaddr     = buf->vaddr;
1079
1080                flb.offset      = flq->un.fl.offset;
1081                lastlen         = min(bufsz, len);
1082                fbuf->len       = lastlen;
1083
1084                len -= lastlen;
1085                if (!len)
1086                        break;
1087                csio_wr_inval_flq_buf(hw, flq);
1088        }
1089
1090        flb.defer_free = flq->un.fl.packen ? 0 : 1;
1091
1092        iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),
1093                   &flb, priv);
1094
1095        if (flq->un.fl.packen)
1096                flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);
1097        else
1098                csio_wr_inval_flq_buf(hw, flq);
1099
1100}
1101
1102/*
1103 * csio_is_new_iqwr - Is this a new Ingress queue entry ?
1104 * @q: Ingress quueue.
1105 * @ftr: Ingress queue WR SGE footer.
1106 *
1107 * The entry is new if our generation bit matches the corresponding
1108 * bit in the footer of the current WR.
1109 */
1110static inline bool
1111csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)
1112{
1113        return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));
1114}
1115
1116/*
1117 * csio_wr_process_iq - Process elements in Ingress queue.
1118 * @hw:  HW pointer
1119 * @qidx: Index of queue
1120 * @iq_handler: Handler for this queue
1121 * @priv: Caller's private pointer
1122 *
1123 * This routine walks through every entry of the ingress queue, calling
1124 * the provided iq_handler with the entry, until the generation bit
1125 * flips.
1126 */
1127int
1128csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,
1129                   void (*iq_handler)(struct csio_hw *, void *,
1130                                      uint32_t, struct csio_fl_dma_buf *,
1131                                      void *),
1132                   void *priv)
1133{
1134        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1135        void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));
1136        struct csio_iqwr_footer *ftr;
1137        uint32_t wr_type, fw_qid, qid;
1138        struct csio_q *q_completed;
1139        struct csio_q *flq = csio_iq_has_fl(q) ?
1140                                        wrm->q_arr[q->un.iq.flq_idx] : NULL;
1141        int rv = 0;
1142
1143        /* Get the footer */
1144        ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1145                                          (q->wr_sz - sizeof(*ftr)));
1146
1147        /*
1148         * When q wrapped around last time, driver should have inverted
1149         * ic.genbit as well.
1150         */
1151        while (csio_is_new_iqwr(q, ftr)) {
1152
1153                CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=
1154                                                (uintptr_t)q->vwrap);
1155                rmb();
1156                wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);
1157
1158                switch (wr_type) {
1159                case X_RSPD_TYPE_CPL:
1160                        /* Subtract footer from WR len */
1161                        iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);
1162                        break;
1163                case X_RSPD_TYPE_FLBUF:
1164                        csio_wr_process_fl(hw, q, wr,
1165                                           ntohl(ftr->pldbuflen_qid),
1166                                           iq_handler, priv);
1167                        break;
1168                case X_RSPD_TYPE_INTR:
1169                        fw_qid = ntohl(ftr->pldbuflen_qid);
1170                        qid = fw_qid - wrm->fw_iq_start;
1171                        q_completed = hw->wrm.intr_map[qid];
1172
1173                        if (unlikely(qid ==
1174                                        csio_q_physiqid(hw, hw->intr_iq_idx))) {
1175                                /*
1176                                 * We are already in the Forward Interrupt
1177                                 * Interrupt Queue Service! Do-not service
1178                                 * again!
1179                                 *
1180                                 */
1181                        } else {
1182                                CSIO_DB_ASSERT(q_completed);
1183                                CSIO_DB_ASSERT(
1184                                        q_completed->un.iq.iq_intx_handler);
1185
1186                                /* Call the queue handler. */
1187                                q_completed->un.iq.iq_intx_handler(hw, NULL,
1188                                                0, NULL, (void *)q_completed);
1189                        }
1190                        break;
1191                default:
1192                        csio_warn(hw, "Unknown resp type 0x%x received\n",
1193                                 wr_type);
1194                        CSIO_INC_STATS(q, n_rsp_unknown);
1195                        break;
1196                }
1197
1198                /*
1199                 * Ingress *always* has fixed size WR entries. Therefore,
1200                 * there should always be complete WRs towards the end of
1201                 * queue.
1202                 */
1203                if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {
1204
1205                        /* Roll over to start of queue */
1206                        q->cidx = 0;
1207                        wr      = q->vstart;
1208
1209                        /* Toggle genbit */
1210                        q->un.iq.genbit ^= 0x1;
1211
1212                        CSIO_INC_STATS(q, n_qwrap);
1213                } else {
1214                        q->cidx++;
1215                        wr      = (void *)((uintptr_t)(q->vstart) +
1216                                           (q->cidx * q->wr_sz));
1217                }
1218
1219                ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1220                                                  (q->wr_sz - sizeof(*ftr)));
1221                q->inc_idx++;
1222
1223        } /* while (q->un.iq.genbit == hdr->genbit) */
1224
1225        /*
1226         * We need to re-arm SGE interrupts in case we got a stray interrupt,
1227         * especially in msix mode. With INTx, this may be a common occurence.
1228         */
1229        if (unlikely(!q->inc_idx)) {
1230                CSIO_INC_STATS(q, n_stray_comp);
1231                rv = -EINVAL;
1232                goto restart;
1233        }
1234
1235        /* Replenish free list buffers if pending falls below low water mark */
1236        if (flq) {
1237                uint32_t avail  = csio_wr_avail_qcredits(flq);
1238                if (avail <= 16) {
1239                        /* Make sure in FLQ, atleast 1 credit (8 FL buffers)
1240                         * remains unpopulated otherwise HW thinks
1241                         * FLQ is empty.
1242                         */
1243                        csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);
1244                        csio_wr_ring_fldb(hw, flq);
1245                }
1246        }
1247
1248restart:
1249        /* Now inform SGE about our incremental index value */
1250        csio_wr_reg32(hw, CIDXINC(q->inc_idx)           |
1251                          INGRESSQID(q->un.iq.physiqid) |
1252                          TIMERREG(csio_sge_timer_reg),
1253                          MYPF_REG(SGE_PF_GTS));
1254        q->stats.n_tot_rsps += q->inc_idx;
1255
1256        q->inc_idx = 0;
1257
1258        return rv;
1259}
1260
1261int
1262csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,
1263                   void (*iq_handler)(struct csio_hw *, void *,
1264                                      uint32_t, struct csio_fl_dma_buf *,
1265                                      void *),
1266                   void *priv)
1267{
1268        struct csio_wrm *wrm    = csio_hw_to_wrm(hw);
1269        struct csio_q   *iq     = wrm->q_arr[qidx];
1270
1271        return csio_wr_process_iq(hw, iq, iq_handler, priv);
1272}
1273
1274static int
1275csio_closest_timer(struct csio_sge *s, int time)
1276{
1277        int i, delta, match = 0, min_delta = INT_MAX;
1278
1279        for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1280                delta = time - s->timer_val[i];
1281                if (delta < 0)
1282                        delta = -delta;
1283                if (delta < min_delta) {
1284                        min_delta = delta;
1285                        match = i;
1286                }
1287        }
1288        return match;
1289}
1290
1291static int
1292csio_closest_thresh(struct csio_sge *s, int cnt)
1293{
1294        int i, delta, match = 0, min_delta = INT_MAX;
1295
1296        for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1297                delta = cnt - s->counter_val[i];
1298                if (delta < 0)
1299                        delta = -delta;
1300                if (delta < min_delta) {
1301                        min_delta = delta;
1302                        match = i;
1303                }
1304        }
1305        return match;
1306}
1307
1308static void
1309csio_wr_fixup_host_params(struct csio_hw *hw)
1310{
1311        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1312        struct csio_sge *sge = &wrm->sge;
1313        uint32_t clsz = L1_CACHE_BYTES;
1314        uint32_t s_hps = PAGE_SHIFT - 10;
1315        uint32_t ingpad = 0;
1316        uint32_t stat_len = clsz > 64 ? 128 : 64;
1317
1318        csio_wr_reg32(hw, HOSTPAGESIZEPF0(s_hps) | HOSTPAGESIZEPF1(s_hps) |
1319                      HOSTPAGESIZEPF2(s_hps) | HOSTPAGESIZEPF3(s_hps) |
1320                      HOSTPAGESIZEPF4(s_hps) | HOSTPAGESIZEPF5(s_hps) |
1321                      HOSTPAGESIZEPF6(s_hps) | HOSTPAGESIZEPF7(s_hps),
1322                      SGE_HOST_PAGE_SIZE);
1323
1324        sge->csio_fl_align = clsz < 32 ? 32 : clsz;
1325        ingpad = ilog2(sge->csio_fl_align) - 5;
1326
1327        csio_set_reg_field(hw, SGE_CONTROL, INGPADBOUNDARY_MASK |
1328                                            EGRSTATUSPAGESIZE(1),
1329                           INGPADBOUNDARY(ingpad) |
1330                           EGRSTATUSPAGESIZE(stat_len != 64));
1331
1332        /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */
1333        csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0);
1334        csio_wr_reg32(hw,
1335                      (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2) +
1336                      sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
1337                      SGE_FL_BUFFER_SIZE2);
1338        csio_wr_reg32(hw,
1339                      (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3) +
1340                      sge->csio_fl_align - 1) & ~(sge->csio_fl_align - 1),
1341                      SGE_FL_BUFFER_SIZE3);
1342
1343        csio_wr_reg32(hw, HPZ0(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ);
1344
1345        /* default value of rx_dma_offset of the NIC driver */
1346        csio_set_reg_field(hw, SGE_CONTROL, PKTSHIFT_MASK,
1347                           PKTSHIFT(CSIO_SGE_RX_DMA_OFFSET));
1348}
1349
1350static void
1351csio_init_intr_coalesce_parms(struct csio_hw *hw)
1352{
1353        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1354        struct csio_sge *sge = &wrm->sge;
1355
1356        csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);
1357        if (csio_intr_coalesce_cnt) {
1358                csio_sge_thresh_reg = 0;
1359                csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;
1360                return;
1361        }
1362
1363        csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);
1364}
1365
1366/*
1367 * csio_wr_get_sge - Get SGE register values.
1368 * @hw: HW module.
1369 *
1370 * Used by non-master functions and by master-functions relying on config file.
1371 */
1372static void
1373csio_wr_get_sge(struct csio_hw *hw)
1374{
1375        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1376        struct csio_sge *sge = &wrm->sge;
1377        uint32_t ingpad;
1378        int i;
1379        u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
1380        u32 ingress_rx_threshold;
1381
1382        sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL);
1383
1384        ingpad = INGPADBOUNDARY_GET(sge->sge_control);
1385
1386        switch (ingpad) {
1387        case X_INGPCIEBOUNDARY_32B:
1388                sge->csio_fl_align = 32; break;
1389        case X_INGPCIEBOUNDARY_64B:
1390                sge->csio_fl_align = 64; break;
1391        case X_INGPCIEBOUNDARY_128B:
1392                sge->csio_fl_align = 128; break;
1393        case X_INGPCIEBOUNDARY_256B:
1394                sge->csio_fl_align = 256; break;
1395        case X_INGPCIEBOUNDARY_512B:
1396                sge->csio_fl_align = 512; break;
1397        case X_INGPCIEBOUNDARY_1024B:
1398                sge->csio_fl_align = 1024; break;
1399        case X_INGPCIEBOUNDARY_2048B:
1400                sge->csio_fl_align = 2048; break;
1401        case X_INGPCIEBOUNDARY_4096B:
1402                sge->csio_fl_align = 4096; break;
1403        }
1404
1405        for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1406                csio_get_flbuf_size(hw, sge, i);
1407
1408        timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1);
1409        timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3);
1410        timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5);
1411
1412        sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,
1413                                        TIMERVALUE0_GET(timer_value_0_and_1));
1414        sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,
1415                                        TIMERVALUE1_GET(timer_value_0_and_1));
1416        sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,
1417                                        TIMERVALUE2_GET(timer_value_2_and_3));
1418        sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,
1419                                        TIMERVALUE3_GET(timer_value_2_and_3));
1420        sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,
1421                                        TIMERVALUE4_GET(timer_value_4_and_5));
1422        sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,
1423                                        TIMERVALUE5_GET(timer_value_4_and_5));
1424
1425        ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD);
1426        sge->counter_val[0] = THRESHOLD_0_GET(ingress_rx_threshold);
1427        sge->counter_val[1] = THRESHOLD_1_GET(ingress_rx_threshold);
1428        sge->counter_val[2] = THRESHOLD_2_GET(ingress_rx_threshold);
1429        sge->counter_val[3] = THRESHOLD_3_GET(ingress_rx_threshold);
1430
1431        csio_init_intr_coalesce_parms(hw);
1432}
1433
1434/*
1435 * csio_wr_set_sge - Initialize SGE registers
1436 * @hw: HW module.
1437 *
1438 * Used by Master function to initialize SGE registers in the absence
1439 * of a config file.
1440 */
1441static void
1442csio_wr_set_sge(struct csio_hw *hw)
1443{
1444        struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1445        struct csio_sge *sge = &wrm->sge;
1446        int i;
1447
1448        /*
1449         * Set up our basic SGE mode to deliver CPL messages to our Ingress
1450         * Queue and Packet Date to the Free List.
1451         */
1452        csio_set_reg_field(hw, SGE_CONTROL, RXPKTCPLMODE(1), RXPKTCPLMODE(1));
1453
1454        sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL);
1455
1456        /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */
1457
1458        /*
1459         * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows
1460         * and generate an interrupt when this occurs so we can recover.
1461         */
1462        csio_set_reg_field(hw, SGE_DBFIFO_STATUS,
1463                           HP_INT_THRESH(HP_INT_THRESH_MASK) |
1464                           LP_INT_THRESH(LP_INT_THRESH_MASK),
1465                           HP_INT_THRESH(CSIO_SGE_DBFIFO_INT_THRESH) |
1466                           LP_INT_THRESH(CSIO_SGE_DBFIFO_INT_THRESH));
1467        csio_set_reg_field(hw, SGE_DOORBELL_CONTROL, ENABLE_DROP,
1468                           ENABLE_DROP);
1469
1470        /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */
1471
1472        CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);
1473        CSIO_SET_FLBUF_SIZE(hw, 2, CSIO_SGE_FLBUF_SIZE2);
1474        CSIO_SET_FLBUF_SIZE(hw, 3, CSIO_SGE_FLBUF_SIZE3);
1475        CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);
1476        CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);
1477        CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);
1478        CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);
1479        CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);
1480
1481        for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1482                csio_get_flbuf_size(hw, sge, i);
1483
1484        /* Initialize interrupt coalescing attributes */
1485        sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;
1486        sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;
1487        sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;
1488        sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;
1489        sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;
1490        sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;
1491
1492        sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;
1493        sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;
1494        sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;
1495        sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;
1496
1497        csio_wr_reg32(hw, THRESHOLD_0(sge->counter_val[0]) |
1498                      THRESHOLD_1(sge->counter_val[1]) |
1499                      THRESHOLD_2(sge->counter_val[2]) |
1500                      THRESHOLD_3(sge->counter_val[3]),
1501                      SGE_INGRESS_RX_THRESHOLD);
1502
1503        csio_wr_reg32(hw,
1504                   TIMERVALUE0(csio_us_to_core_ticks(hw, sge->timer_val[0])) |
1505                   TIMERVALUE1(csio_us_to_core_ticks(hw, sge->timer_val[1])),
1506                   SGE_TIMER_VALUE_0_AND_1);
1507
1508        csio_wr_reg32(hw,
1509                   TIMERVALUE2(csio_us_to_core_ticks(hw, sge->timer_val[2])) |
1510                   TIMERVALUE3(csio_us_to_core_ticks(hw, sge->timer_val[3])),
1511                   SGE_TIMER_VALUE_2_AND_3);
1512
1513        csio_wr_reg32(hw,
1514                   TIMERVALUE4(csio_us_to_core_ticks(hw, sge->timer_val[4])) |
1515                   TIMERVALUE5(csio_us_to_core_ticks(hw, sge->timer_val[5])),
1516                   SGE_TIMER_VALUE_4_AND_5);
1517
1518        csio_init_intr_coalesce_parms(hw);
1519}
1520
1521void
1522csio_wr_sge_init(struct csio_hw *hw)
1523{
1524        /*
1525         * If we are master:
1526         *    - If we plan to use the config file, we need to fixup some
1527         *      host specific registers, and read the rest of the SGE
1528         *      configuration.
1529         *    - If we dont plan to use the config file, we need to initialize
1530         *      SGE entirely, including fixing the host specific registers.
1531         * If we arent the master, we are only allowed to read and work off of
1532         *      the already initialized SGE values.
1533         *
1534         * Therefore, before calling this function, we assume that the master-
1535         * ship of the card, and whether to use config file or not, have
1536         * already been decided. In other words, CSIO_HWF_USING_SOFT_PARAMS and
1537         * CSIO_HWF_MASTER should be set/unset.
1538         */
1539        if (csio_is_hw_master(hw)) {
1540                csio_wr_fixup_host_params(hw);
1541
1542                if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)
1543                        csio_wr_get_sge(hw);
1544                else
1545                        csio_wr_set_sge(hw);
1546        } else
1547                csio_wr_get_sge(hw);
1548}
1549
1550/*
1551 * csio_wrm_init - Initialize Work request module.
1552 * @wrm: WR module
1553 * @hw: HW pointer
1554 *
1555 * Allocates memory for an array of queue pointers starting at q_arr.
1556 */
1557int
1558csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)
1559{
1560        int i;
1561
1562        if (!wrm->num_q) {
1563                csio_err(hw, "Num queues is not set\n");
1564                return -EINVAL;
1565        }
1566
1567        wrm->q_arr = kzalloc(sizeof(struct csio_q *) * wrm->num_q, GFP_KERNEL);
1568        if (!wrm->q_arr)
1569                goto err;
1570
1571        for (i = 0; i < wrm->num_q; i++) {
1572                wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);
1573                if (!wrm->q_arr[i]) {
1574                        while (--i >= 0)
1575                                kfree(wrm->q_arr[i]);
1576                        goto err_free_arr;
1577                }
1578        }
1579        wrm->free_qidx  = 0;
1580
1581        return 0;
1582
1583err_free_arr:
1584        kfree(wrm->q_arr);
1585err:
1586        return -ENOMEM;
1587}
1588
1589/*
1590 * csio_wrm_exit - Initialize Work request module.
1591 * @wrm: WR module
1592 * @hw: HW module
1593 *
1594 * Uninitialize WR module. Free q_arr and pointers in it.
1595 * We have the additional job of freeing the DMA memory associated
1596 * with the queues.
1597 */
1598void
1599csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)
1600{
1601        int i;
1602        uint32_t j;
1603        struct csio_q *q;
1604        struct csio_dma_buf *buf;
1605
1606        for (i = 0; i < wrm->num_q; i++) {
1607                q = wrm->q_arr[i];
1608
1609                if (wrm->free_qidx && (i < wrm->free_qidx)) {
1610                        if (q->type == CSIO_FREELIST) {
1611                                if (!q->un.fl.bufs)
1612                                        continue;
1613                                for (j = 0; j < q->credits; j++) {
1614                                        buf = &q->un.fl.bufs[j];
1615                                        if (!buf->vaddr)
1616                                                continue;
1617                                        pci_free_consistent(hw->pdev, buf->len,
1618                                                            buf->vaddr,
1619                                                            buf->paddr);
1620                                }
1621                                kfree(q->un.fl.bufs);
1622                        }
1623                        pci_free_consistent(hw->pdev, q->size,
1624                                            q->vstart, q->pstart);
1625                }
1626                kfree(q);
1627        }
1628
1629        hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;
1630
1631        kfree(wrm->q_arr);
1632}
1633