linux/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
<<
>>
Prefs
   1/*
   2 * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet
   3 * driver for Linux.
   4 *
   5 * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved.
   6 *
   7 * This software is available to you under a choice of one of two
   8 * licenses.  You may choose to be licensed under the terms of the GNU
   9 * General Public License (GPL) Version 2, available from the file
  10 * COPYING in the main directory of this source tree, or the
  11 * OpenIB.org BSD license below:
  12 *
  13 *     Redistribution and use in source and binary forms, with or
  14 *     without modification, are permitted provided that the following
  15 *     conditions are met:
  16 *
  17 *      - Redistributions of source code must retain the above
  18 *        copyright notice, this list of conditions and the following
  19 *        disclaimer.
  20 *
  21 *      - Redistributions in binary form must reproduce the above
  22 *        copyright notice, this list of conditions and the following
  23 *        disclaimer in the documentation and/or other materials
  24 *        provided with the distribution.
  25 *
  26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33 * SOFTWARE.
  34 */
  35
  36#include <linux/skbuff.h>
  37#include <linux/netdevice.h>
  38#include <linux/etherdevice.h>
  39#include <linux/if_vlan.h>
  40#include <linux/ip.h>
  41#include <net/ipv6.h>
  42#include <net/tcp.h>
  43#include <linux/dma-mapping.h>
  44#include <linux/prefetch.h>
  45
  46#include "t4vf_common.h"
  47#include "t4vf_defs.h"
  48
  49#include "../cxgb4/t4_regs.h"
  50#include "../cxgb4/t4_values.h"
  51#include "../cxgb4/t4fw_api.h"
  52#include "../cxgb4/t4_msg.h"
  53
  54/*
  55 * Constants ...
  56 */
  57enum {
  58        /*
  59         * Egress Queue sizes, producer and consumer indices are all in units
  60         * of Egress Context Units bytes.  Note that as far as the hardware is
  61         * concerned, the free list is an Egress Queue (the host produces free
  62         * buffers which the hardware consumes) and free list entries are
  63         * 64-bit PCI DMA addresses.
  64         */
  65        EQ_UNIT = SGE_EQ_IDXSIZE,
  66        FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
  67        TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
  68
  69        /*
  70         * Max number of TX descriptors we clean up at a time.  Should be
  71         * modest as freeing skbs isn't cheap and it happens while holding
  72         * locks.  We just need to free packets faster than they arrive, we
  73         * eventually catch up and keep the amortized cost reasonable.
  74         */
  75        MAX_TX_RECLAIM = 16,
  76
  77        /*
  78         * Max number of Rx buffers we replenish at a time.  Again keep this
  79         * modest, allocating buffers isn't cheap either.
  80         */
  81        MAX_RX_REFILL = 16,
  82
  83        /*
  84         * Period of the Rx queue check timer.  This timer is infrequent as it
  85         * has something to do only when the system experiences severe memory
  86         * shortage.
  87         */
  88        RX_QCHECK_PERIOD = (HZ / 2),
  89
  90        /*
  91         * Period of the TX queue check timer and the maximum number of TX
  92         * descriptors to be reclaimed by the TX timer.
  93         */
  94        TX_QCHECK_PERIOD = (HZ / 2),
  95        MAX_TIMER_TX_RECLAIM = 100,
  96
  97        /*
  98         * Suspend an Ethernet TX queue with fewer available descriptors than
  99         * this.  We always want to have room for a maximum sized packet:
 100         * inline immediate data + MAX_SKB_FRAGS. This is the same as
 101         * calc_tx_flits() for a TSO packet with nr_frags == MAX_SKB_FRAGS
 102         * (see that function and its helpers for a description of the
 103         * calculation).
 104         */
 105        ETHTXQ_MAX_FRAGS = MAX_SKB_FRAGS + 1,
 106        ETHTXQ_MAX_SGL_LEN = ((3 * (ETHTXQ_MAX_FRAGS-1))/2 +
 107                                   ((ETHTXQ_MAX_FRAGS-1) & 1) +
 108                                   2),
 109        ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) +
 110                          sizeof(struct cpl_tx_pkt_lso_core) +
 111                          sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64),
 112        ETHTXQ_MAX_FLITS = ETHTXQ_MAX_SGL_LEN + ETHTXQ_MAX_HDR,
 113
 114        ETHTXQ_STOP_THRES = 1 + DIV_ROUND_UP(ETHTXQ_MAX_FLITS, TXD_PER_EQ_UNIT),
 115
 116        /*
 117         * Max TX descriptor space we allow for an Ethernet packet to be
 118         * inlined into a WR.  This is limited by the maximum value which
 119         * we can specify for immediate data in the firmware Ethernet TX
 120         * Work Request.
 121         */
 122        MAX_IMM_TX_PKT_LEN = FW_WR_IMMDLEN_M,
 123
 124        /*
 125         * Max size of a WR sent through a control TX queue.
 126         */
 127        MAX_CTRL_WR_LEN = 256,
 128
 129        /*
 130         * Maximum amount of data which we'll ever need to inline into a
 131         * TX ring: max(MAX_IMM_TX_PKT_LEN, MAX_CTRL_WR_LEN).
 132         */
 133        MAX_IMM_TX_LEN = (MAX_IMM_TX_PKT_LEN > MAX_CTRL_WR_LEN
 134                          ? MAX_IMM_TX_PKT_LEN
 135                          : MAX_CTRL_WR_LEN),
 136
 137        /*
 138         * For incoming packets less than RX_COPY_THRES, we copy the data into
 139         * an skb rather than referencing the data.  We allocate enough
 140         * in-line room in skb's to accommodate pulling in RX_PULL_LEN bytes
 141         * of the data (header).
 142         */
 143        RX_COPY_THRES = 256,
 144        RX_PULL_LEN = 128,
 145
 146        /*
 147         * Main body length for sk_buffs used for RX Ethernet packets with
 148         * fragments.  Should be >= RX_PULL_LEN but possibly bigger to give
 149         * pskb_may_pull() some room.
 150         */
 151        RX_SKB_LEN = 512,
 152};
 153
 154/*
 155 * Software state per TX descriptor.
 156 */
 157struct tx_sw_desc {
 158        struct sk_buff *skb;            /* socket buffer of TX data source */
 159        struct ulptx_sgl *sgl;          /* scatter/gather list in TX Queue */
 160};
 161
 162/*
 163 * Software state per RX Free List descriptor.  We keep track of the allocated
 164 * FL page, its size, and its PCI DMA address (if the page is mapped).  The FL
 165 * page size and its PCI DMA mapped state are stored in the low bits of the
 166 * PCI DMA address as per below.
 167 */
 168struct rx_sw_desc {
 169        struct page *page;              /* Free List page buffer */
 170        dma_addr_t dma_addr;            /* PCI DMA address (if mapped) */
 171                                        /*   and flags (see below) */
 172};
 173
 174/*
 175 * The low bits of rx_sw_desc.dma_addr have special meaning.  Note that the
 176 * SGE also uses the low 4 bits to determine the size of the buffer.  It uses
 177 * those bits to index into the SGE_FL_BUFFER_SIZE[index] register array.
 178 * Since we only use SGE_FL_BUFFER_SIZE0 and SGE_FL_BUFFER_SIZE1, these low 4
 179 * bits can only contain a 0 or a 1 to indicate which size buffer we're giving
 180 * to the SGE.  Thus, our software state of "is the buffer mapped for DMA" is
 181 * maintained in an inverse sense so the hardware never sees that bit high.
 182 */
 183enum {
 184        RX_LARGE_BUF    = 1 << 0,       /* buffer is SGE_FL_BUFFER_SIZE[1] */
 185        RX_UNMAPPED_BUF = 1 << 1,       /* buffer is not mapped */
 186};
 187
 188/**
 189 *      get_buf_addr - return DMA buffer address of software descriptor
 190 *      @sdesc: pointer to the software buffer descriptor
 191 *
 192 *      Return the DMA buffer address of a software descriptor (stripping out
 193 *      our low-order flag bits).
 194 */
 195static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *sdesc)
 196{
 197        return sdesc->dma_addr & ~(dma_addr_t)(RX_LARGE_BUF | RX_UNMAPPED_BUF);
 198}
 199
 200/**
 201 *      is_buf_mapped - is buffer mapped for DMA?
 202 *      @sdesc: pointer to the software buffer descriptor
 203 *
 204 *      Determine whether the buffer associated with a software descriptor in
 205 *      mapped for DMA or not.
 206 */
 207static inline bool is_buf_mapped(const struct rx_sw_desc *sdesc)
 208{
 209        return !(sdesc->dma_addr & RX_UNMAPPED_BUF);
 210}
 211
 212/**
 213 *      need_skb_unmap - does the platform need unmapping of sk_buffs?
 214 *
 215 *      Returns true if the platform needs sk_buff unmapping.  The compiler
 216 *      optimizes away unnecessary code if this returns true.
 217 */
 218static inline int need_skb_unmap(void)
 219{
 220#ifdef CONFIG_NEED_DMA_MAP_STATE
 221        return 1;
 222#else
 223        return 0;
 224#endif
 225}
 226
 227/**
 228 *      txq_avail - return the number of available slots in a TX queue
 229 *      @tq: the TX queue
 230 *
 231 *      Returns the number of available descriptors in a TX queue.
 232 */
 233static inline unsigned int txq_avail(const struct sge_txq *tq)
 234{
 235        return tq->size - 1 - tq->in_use;
 236}
 237
 238/**
 239 *      fl_cap - return the capacity of a Free List
 240 *      @fl: the Free List
 241 *
 242 *      Returns the capacity of a Free List.  The capacity is less than the
 243 *      size because an Egress Queue Index Unit worth of descriptors needs to
 244 *      be left unpopulated, otherwise the Producer and Consumer indices PIDX
 245 *      and CIDX will match and the hardware will think the FL is empty.
 246 */
 247static inline unsigned int fl_cap(const struct sge_fl *fl)
 248{
 249        return fl->size - FL_PER_EQ_UNIT;
 250}
 251
 252/**
 253 *      fl_starving - return whether a Free List is starving.
 254 *      @adapter: pointer to the adapter
 255 *      @fl: the Free List
 256 *
 257 *      Tests specified Free List to see whether the number of buffers
 258 *      available to the hardware has falled below our "starvation"
 259 *      threshold.
 260 */
 261static inline bool fl_starving(const struct adapter *adapter,
 262                               const struct sge_fl *fl)
 263{
 264        const struct sge *s = &adapter->sge;
 265
 266        return fl->avail - fl->pend_cred <= s->fl_starve_thres;
 267}
 268
 269/**
 270 *      map_skb -  map an skb for DMA to the device
 271 *      @dev: the egress net device
 272 *      @skb: the packet to map
 273 *      @addr: a pointer to the base of the DMA mapping array
 274 *
 275 *      Map an skb for DMA to the device and return an array of DMA addresses.
 276 */
 277static int map_skb(struct device *dev, const struct sk_buff *skb,
 278                   dma_addr_t *addr)
 279{
 280        const skb_frag_t *fp, *end;
 281        const struct skb_shared_info *si;
 282
 283        *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
 284        if (dma_mapping_error(dev, *addr))
 285                goto out_err;
 286
 287        si = skb_shinfo(skb);
 288        end = &si->frags[si->nr_frags];
 289        for (fp = si->frags; fp < end; fp++) {
 290                *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp),
 291                                           DMA_TO_DEVICE);
 292                if (dma_mapping_error(dev, *addr))
 293                        goto unwind;
 294        }
 295        return 0;
 296
 297unwind:
 298        while (fp-- > si->frags)
 299                dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
 300        dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE);
 301
 302out_err:
 303        return -ENOMEM;
 304}
 305
 306static void unmap_sgl(struct device *dev, const struct sk_buff *skb,
 307                      const struct ulptx_sgl *sgl, const struct sge_txq *tq)
 308{
 309        const struct ulptx_sge_pair *p;
 310        unsigned int nfrags = skb_shinfo(skb)->nr_frags;
 311
 312        if (likely(skb_headlen(skb)))
 313                dma_unmap_single(dev, be64_to_cpu(sgl->addr0),
 314                                 be32_to_cpu(sgl->len0), DMA_TO_DEVICE);
 315        else {
 316                dma_unmap_page(dev, be64_to_cpu(sgl->addr0),
 317                               be32_to_cpu(sgl->len0), DMA_TO_DEVICE);
 318                nfrags--;
 319        }
 320
 321        /*
 322         * the complexity below is because of the possibility of a wrap-around
 323         * in the middle of an SGL
 324         */
 325        for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
 326                if (likely((u8 *)(p + 1) <= (u8 *)tq->stat)) {
 327unmap:
 328                        dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
 329                                       be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
 330                        dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
 331                                       be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
 332                        p++;
 333                } else if ((u8 *)p == (u8 *)tq->stat) {
 334                        p = (const struct ulptx_sge_pair *)tq->desc;
 335                        goto unmap;
 336                } else if ((u8 *)p + 8 == (u8 *)tq->stat) {
 337                        const __be64 *addr = (const __be64 *)tq->desc;
 338
 339                        dma_unmap_page(dev, be64_to_cpu(addr[0]),
 340                                       be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
 341                        dma_unmap_page(dev, be64_to_cpu(addr[1]),
 342                                       be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
 343                        p = (const struct ulptx_sge_pair *)&addr[2];
 344                } else {
 345                        const __be64 *addr = (const __be64 *)tq->desc;
 346
 347                        dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
 348                                       be32_to_cpu(p->len[0]), DMA_TO_DEVICE);
 349                        dma_unmap_page(dev, be64_to_cpu(addr[0]),
 350                                       be32_to_cpu(p->len[1]), DMA_TO_DEVICE);
 351                        p = (const struct ulptx_sge_pair *)&addr[1];
 352                }
 353        }
 354        if (nfrags) {
 355                __be64 addr;
 356
 357                if ((u8 *)p == (u8 *)tq->stat)
 358                        p = (const struct ulptx_sge_pair *)tq->desc;
 359                addr = ((u8 *)p + 16 <= (u8 *)tq->stat
 360                        ? p->addr[0]
 361                        : *(const __be64 *)tq->desc);
 362                dma_unmap_page(dev, be64_to_cpu(addr), be32_to_cpu(p->len[0]),
 363                               DMA_TO_DEVICE);
 364        }
 365}
 366
 367/**
 368 *      free_tx_desc - reclaims TX descriptors and their buffers
 369 *      @adapter: the adapter
 370 *      @tq: the TX queue to reclaim descriptors from
 371 *      @n: the number of descriptors to reclaim
 372 *      @unmap: whether the buffers should be unmapped for DMA
 373 *
 374 *      Reclaims TX descriptors from an SGE TX queue and frees the associated
 375 *      TX buffers.  Called with the TX queue lock held.
 376 */
 377static void free_tx_desc(struct adapter *adapter, struct sge_txq *tq,
 378                         unsigned int n, bool unmap)
 379{
 380        struct tx_sw_desc *sdesc;
 381        unsigned int cidx = tq->cidx;
 382        struct device *dev = adapter->pdev_dev;
 383
 384        const int need_unmap = need_skb_unmap() && unmap;
 385
 386        sdesc = &tq->sdesc[cidx];
 387        while (n--) {
 388                /*
 389                 * If we kept a reference to the original TX skb, we need to
 390                 * unmap it from PCI DMA space (if required) and free it.
 391                 */
 392                if (sdesc->skb) {
 393                        if (need_unmap)
 394                                unmap_sgl(dev, sdesc->skb, sdesc->sgl, tq);
 395                        dev_consume_skb_any(sdesc->skb);
 396                        sdesc->skb = NULL;
 397                }
 398
 399                sdesc++;
 400                if (++cidx == tq->size) {
 401                        cidx = 0;
 402                        sdesc = tq->sdesc;
 403                }
 404        }
 405        tq->cidx = cidx;
 406}
 407
 408/*
 409 * Return the number of reclaimable descriptors in a TX queue.
 410 */
 411static inline int reclaimable(const struct sge_txq *tq)
 412{
 413        int hw_cidx = be16_to_cpu(tq->stat->cidx);
 414        int reclaimable = hw_cidx - tq->cidx;
 415        if (reclaimable < 0)
 416                reclaimable += tq->size;
 417        return reclaimable;
 418}
 419
 420/**
 421 *      reclaim_completed_tx - reclaims completed TX descriptors
 422 *      @adapter: the adapter
 423 *      @tq: the TX queue to reclaim completed descriptors from
 424 *      @unmap: whether the buffers should be unmapped for DMA
 425 *
 426 *      Reclaims TX descriptors that the SGE has indicated it has processed,
 427 *      and frees the associated buffers if possible.  Called with the TX
 428 *      queue locked.
 429 */
 430static inline void reclaim_completed_tx(struct adapter *adapter,
 431                                        struct sge_txq *tq,
 432                                        bool unmap)
 433{
 434        int avail = reclaimable(tq);
 435
 436        if (avail) {
 437                /*
 438                 * Limit the amount of clean up work we do at a time to keep
 439                 * the TX lock hold time O(1).
 440                 */
 441                if (avail > MAX_TX_RECLAIM)
 442                        avail = MAX_TX_RECLAIM;
 443
 444                free_tx_desc(adapter, tq, avail, unmap);
 445                tq->in_use -= avail;
 446        }
 447}
 448
 449/**
 450 *      get_buf_size - return the size of an RX Free List buffer.
 451 *      @adapter: pointer to the associated adapter
 452 *      @sdesc: pointer to the software buffer descriptor
 453 */
 454static inline int get_buf_size(const struct adapter *adapter,
 455                               const struct rx_sw_desc *sdesc)
 456{
 457        const struct sge *s = &adapter->sge;
 458
 459        return (s->fl_pg_order > 0 && (sdesc->dma_addr & RX_LARGE_BUF)
 460                ? (PAGE_SIZE << s->fl_pg_order) : PAGE_SIZE);
 461}
 462
 463/**
 464 *      free_rx_bufs - free RX buffers on an SGE Free List
 465 *      @adapter: the adapter
 466 *      @fl: the SGE Free List to free buffers from
 467 *      @n: how many buffers to free
 468 *
 469 *      Release the next @n buffers on an SGE Free List RX queue.   The
 470 *      buffers must be made inaccessible to hardware before calling this
 471 *      function.
 472 */
 473static void free_rx_bufs(struct adapter *adapter, struct sge_fl *fl, int n)
 474{
 475        while (n--) {
 476                struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx];
 477
 478                if (is_buf_mapped(sdesc))
 479                        dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),
 480                                       get_buf_size(adapter, sdesc),
 481                                       PCI_DMA_FROMDEVICE);
 482                put_page(sdesc->page);
 483                sdesc->page = NULL;
 484                if (++fl->cidx == fl->size)
 485                        fl->cidx = 0;
 486                fl->avail--;
 487        }
 488}
 489
 490/**
 491 *      unmap_rx_buf - unmap the current RX buffer on an SGE Free List
 492 *      @adapter: the adapter
 493 *      @fl: the SGE Free List
 494 *
 495 *      Unmap the current buffer on an SGE Free List RX queue.   The
 496 *      buffer must be made inaccessible to HW before calling this function.
 497 *
 498 *      This is similar to @free_rx_bufs above but does not free the buffer.
 499 *      Do note that the FL still loses any further access to the buffer.
 500 *      This is used predominantly to "transfer ownership" of an FL buffer
 501 *      to another entity (typically an skb's fragment list).
 502 */
 503static void unmap_rx_buf(struct adapter *adapter, struct sge_fl *fl)
 504{
 505        struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx];
 506
 507        if (is_buf_mapped(sdesc))
 508                dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc),
 509                               get_buf_size(adapter, sdesc),
 510                               PCI_DMA_FROMDEVICE);
 511        sdesc->page = NULL;
 512        if (++fl->cidx == fl->size)
 513                fl->cidx = 0;
 514        fl->avail--;
 515}
 516
 517/**
 518 *      ring_fl_db - righ doorbell on free list
 519 *      @adapter: the adapter
 520 *      @fl: the Free List whose doorbell should be rung ...
 521 *
 522 *      Tell the Scatter Gather Engine that there are new free list entries
 523 *      available.
 524 */
 525static inline void ring_fl_db(struct adapter *adapter, struct sge_fl *fl)
 526{
 527        u32 val = adapter->params.arch.sge_fl_db;
 528
 529        /* The SGE keeps track of its Producer and Consumer Indices in terms
 530         * of Egress Queue Units so we can only tell it about integral numbers
 531         * of multiples of Free List Entries per Egress Queue Units ...
 532         */
 533        if (fl->pend_cred >= FL_PER_EQ_UNIT) {
 534                if (is_t4(adapter->params.chip))
 535                        val |= PIDX_V(fl->pend_cred / FL_PER_EQ_UNIT);
 536                else
 537                        val |= PIDX_T5_V(fl->pend_cred / FL_PER_EQ_UNIT);
 538
 539                /* Make sure all memory writes to the Free List queue are
 540                 * committed before we tell the hardware about them.
 541                 */
 542                wmb();
 543
 544                /* If we don't have access to the new User Doorbell (T5+), use
 545                 * the old doorbell mechanism; otherwise use the new BAR2
 546                 * mechanism.
 547                 */
 548                if (unlikely(fl->bar2_addr == NULL)) {
 549                        t4_write_reg(adapter,
 550                                     T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,
 551                                     QID_V(fl->cntxt_id) | val);
 552                } else {
 553                        writel(val | QID_V(fl->bar2_qid),
 554                               fl->bar2_addr + SGE_UDB_KDOORBELL);
 555
 556                        /* This Write memory Barrier will force the write to
 557                         * the User Doorbell area to be flushed.
 558                         */
 559                        wmb();
 560                }
 561                fl->pend_cred %= FL_PER_EQ_UNIT;
 562        }
 563}
 564
 565/**
 566 *      set_rx_sw_desc - initialize software RX buffer descriptor
 567 *      @sdesc: pointer to the softwore RX buffer descriptor
 568 *      @page: pointer to the page data structure backing the RX buffer
 569 *      @dma_addr: PCI DMA address (possibly with low-bit flags)
 570 */
 571static inline void set_rx_sw_desc(struct rx_sw_desc *sdesc, struct page *page,
 572                                  dma_addr_t dma_addr)
 573{
 574        sdesc->page = page;
 575        sdesc->dma_addr = dma_addr;
 576}
 577
 578/*
 579 * Support for poisoning RX buffers ...
 580 */
 581#define POISON_BUF_VAL -1
 582
 583static inline void poison_buf(struct page *page, size_t sz)
 584{
 585#if POISON_BUF_VAL >= 0
 586        memset(page_address(page), POISON_BUF_VAL, sz);
 587#endif
 588}
 589
 590/**
 591 *      refill_fl - refill an SGE RX buffer ring
 592 *      @adapter: the adapter
 593 *      @fl: the Free List ring to refill
 594 *      @n: the number of new buffers to allocate
 595 *      @gfp: the gfp flags for the allocations
 596 *
 597 *      (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
 598 *      allocated with the supplied gfp flags.  The caller must assure that
 599 *      @n does not exceed the queue's capacity -- i.e. (cidx == pidx) _IN
 600 *      EGRESS QUEUE UNITS_ indicates an empty Free List!  Returns the number
 601 *      of buffers allocated.  If afterwards the queue is found critically low,
 602 *      mark it as starving in the bitmap of starving FLs.
 603 */
 604static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl,
 605                              int n, gfp_t gfp)
 606{
 607        struct sge *s = &adapter->sge;
 608        struct page *page;
 609        dma_addr_t dma_addr;
 610        unsigned int cred = fl->avail;
 611        __be64 *d = &fl->desc[fl->pidx];
 612        struct rx_sw_desc *sdesc = &fl->sdesc[fl->pidx];
 613
 614        /*
 615         * Sanity: ensure that the result of adding n Free List buffers
 616         * won't result in wrapping the SGE's Producer Index around to
 617         * it's Consumer Index thereby indicating an empty Free List ...
 618         */
 619        BUG_ON(fl->avail + n > fl->size - FL_PER_EQ_UNIT);
 620
 621        gfp |= __GFP_NOWARN;
 622
 623        /*
 624         * If we support large pages, prefer large buffers and fail over to
 625         * small pages if we can't allocate large pages to satisfy the refill.
 626         * If we don't support large pages, drop directly into the small page
 627         * allocation code.
 628         */
 629        if (s->fl_pg_order == 0)
 630                goto alloc_small_pages;
 631
 632        while (n) {
 633                page = __dev_alloc_pages(gfp, s->fl_pg_order);
 634                if (unlikely(!page)) {
 635                        /*
 636                         * We've failed inour attempt to allocate a "large
 637                         * page".  Fail over to the "small page" allocation
 638                         * below.
 639                         */
 640                        fl->large_alloc_failed++;
 641                        break;
 642                }
 643                poison_buf(page, PAGE_SIZE << s->fl_pg_order);
 644
 645                dma_addr = dma_map_page(adapter->pdev_dev, page, 0,
 646                                        PAGE_SIZE << s->fl_pg_order,
 647                                        PCI_DMA_FROMDEVICE);
 648                if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {
 649                        /*
 650                         * We've run out of DMA mapping space.  Free up the
 651                         * buffer and return with what we've managed to put
 652                         * into the free list.  We don't want to fail over to
 653                         * the small page allocation below in this case
 654                         * because DMA mapping resources are typically
 655                         * critical resources once they become scarse.
 656                         */
 657                        __free_pages(page, s->fl_pg_order);
 658                        goto out;
 659                }
 660                dma_addr |= RX_LARGE_BUF;
 661                *d++ = cpu_to_be64(dma_addr);
 662
 663                set_rx_sw_desc(sdesc, page, dma_addr);
 664                sdesc++;
 665
 666                fl->avail++;
 667                if (++fl->pidx == fl->size) {
 668                        fl->pidx = 0;
 669                        sdesc = fl->sdesc;
 670                        d = fl->desc;
 671                }
 672                n--;
 673        }
 674
 675alloc_small_pages:
 676        while (n--) {
 677                page = __dev_alloc_page(gfp);
 678                if (unlikely(!page)) {
 679                        fl->alloc_failed++;
 680                        break;
 681                }
 682                poison_buf(page, PAGE_SIZE);
 683
 684                dma_addr = dma_map_page(adapter->pdev_dev, page, 0, PAGE_SIZE,
 685                                       PCI_DMA_FROMDEVICE);
 686                if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) {
 687                        put_page(page);
 688                        break;
 689                }
 690                *d++ = cpu_to_be64(dma_addr);
 691
 692                set_rx_sw_desc(sdesc, page, dma_addr);
 693                sdesc++;
 694
 695                fl->avail++;
 696                if (++fl->pidx == fl->size) {
 697                        fl->pidx = 0;
 698                        sdesc = fl->sdesc;
 699                        d = fl->desc;
 700                }
 701        }
 702
 703out:
 704        /*
 705         * Update our accounting state to incorporate the new Free List
 706         * buffers, tell the hardware about them and return the number of
 707         * buffers which we were able to allocate.
 708         */
 709        cred = fl->avail - cred;
 710        fl->pend_cred += cred;
 711        ring_fl_db(adapter, fl);
 712
 713        if (unlikely(fl_starving(adapter, fl))) {
 714                smp_wmb();
 715                set_bit(fl->cntxt_id, adapter->sge.starving_fl);
 716        }
 717
 718        return cred;
 719}
 720
 721/*
 722 * Refill a Free List to its capacity or the Maximum Refill Increment,
 723 * whichever is smaller ...
 724 */
 725static inline void __refill_fl(struct adapter *adapter, struct sge_fl *fl)
 726{
 727        refill_fl(adapter, fl,
 728                  min((unsigned int)MAX_RX_REFILL, fl_cap(fl) - fl->avail),
 729                  GFP_ATOMIC);
 730}
 731
 732/**
 733 *      alloc_ring - allocate resources for an SGE descriptor ring
 734 *      @dev: the PCI device's core device
 735 *      @nelem: the number of descriptors
 736 *      @hwsize: the size of each hardware descriptor
 737 *      @swsize: the size of each software descriptor
 738 *      @busaddrp: the physical PCI bus address of the allocated ring
 739 *      @swringp: return address pointer for software ring
 740 *      @stat_size: extra space in hardware ring for status information
 741 *
 742 *      Allocates resources for an SGE descriptor ring, such as TX queues,
 743 *      free buffer lists, response queues, etc.  Each SGE ring requires
 744 *      space for its hardware descriptors plus, optionally, space for software
 745 *      state associated with each hardware entry (the metadata).  The function
 746 *      returns three values: the virtual address for the hardware ring (the
 747 *      return value of the function), the PCI bus address of the hardware
 748 *      ring (in *busaddrp), and the address of the software ring (in swringp).
 749 *      Both the hardware and software rings are returned zeroed out.
 750 */
 751static void *alloc_ring(struct device *dev, size_t nelem, size_t hwsize,
 752                        size_t swsize, dma_addr_t *busaddrp, void *swringp,
 753                        size_t stat_size)
 754{
 755        /*
 756         * Allocate the hardware ring and PCI DMA bus address space for said.
 757         */
 758        size_t hwlen = nelem * hwsize + stat_size;
 759        void *hwring = dma_alloc_coherent(dev, hwlen, busaddrp, GFP_KERNEL);
 760
 761        if (!hwring)
 762                return NULL;
 763
 764        /*
 765         * If the caller wants a software ring, allocate it and return a
 766         * pointer to it in *swringp.
 767         */
 768        BUG_ON((swsize != 0) != (swringp != NULL));
 769        if (swsize) {
 770                void *swring = kcalloc(nelem, swsize, GFP_KERNEL);
 771
 772                if (!swring) {
 773                        dma_free_coherent(dev, hwlen, hwring, *busaddrp);
 774                        return NULL;
 775                }
 776                *(void **)swringp = swring;
 777        }
 778
 779        return hwring;
 780}
 781
 782/**
 783 *      sgl_len - calculates the size of an SGL of the given capacity
 784 *      @n: the number of SGL entries
 785 *
 786 *      Calculates the number of flits (8-byte units) needed for a Direct
 787 *      Scatter/Gather List that can hold the given number of entries.
 788 */
 789static inline unsigned int sgl_len(unsigned int n)
 790{
 791        /*
 792         * A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
 793         * addresses.  The DSGL Work Request starts off with a 32-bit DSGL
 794         * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
 795         * repeated sequences of { Length[i], Length[i+1], Address[i],
 796         * Address[i+1] } (this ensures that all addresses are on 64-bit
 797         * boundaries).  If N is even, then Length[N+1] should be set to 0 and
 798         * Address[N+1] is omitted.
 799         *
 800         * The following calculation incorporates all of the above.  It's
 801         * somewhat hard to follow but, briefly: the "+2" accounts for the
 802         * first two flits which include the DSGL header, Length0 and
 803         * Address0; the "(3*(n-1))/2" covers the main body of list entries (3
 804         * flits for every pair of the remaining N) +1 if (n-1) is odd; and
 805         * finally the "+((n-1)&1)" adds the one remaining flit needed if
 806         * (n-1) is odd ...
 807         */
 808        n--;
 809        return (3 * n) / 2 + (n & 1) + 2;
 810}
 811
 812/**
 813 *      flits_to_desc - returns the num of TX descriptors for the given flits
 814 *      @flits: the number of flits
 815 *
 816 *      Returns the number of TX descriptors needed for the supplied number
 817 *      of flits.
 818 */
 819static inline unsigned int flits_to_desc(unsigned int flits)
 820{
 821        BUG_ON(flits > SGE_MAX_WR_LEN / sizeof(__be64));
 822        return DIV_ROUND_UP(flits, TXD_PER_EQ_UNIT);
 823}
 824
 825/**
 826 *      is_eth_imm - can an Ethernet packet be sent as immediate data?
 827 *      @skb: the packet
 828 *
 829 *      Returns whether an Ethernet packet is small enough to fit completely as
 830 *      immediate data.
 831 */
 832static inline int is_eth_imm(const struct sk_buff *skb)
 833{
 834        /*
 835         * The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request
 836         * which does not accommodate immediate data.  We could dike out all
 837         * of the support code for immediate data but that would tie our hands
 838         * too much if we ever want to enhace the firmware.  It would also
 839         * create more differences between the PF and VF Drivers.
 840         */
 841        return false;
 842}
 843
 844/**
 845 *      calc_tx_flits - calculate the number of flits for a packet TX WR
 846 *      @skb: the packet
 847 *
 848 *      Returns the number of flits needed for a TX Work Request for the
 849 *      given Ethernet packet, including the needed WR and CPL headers.
 850 */
 851static inline unsigned int calc_tx_flits(const struct sk_buff *skb)
 852{
 853        unsigned int flits;
 854
 855        /*
 856         * If the skb is small enough, we can pump it out as a work request
 857         * with only immediate data.  In that case we just have to have the
 858         * TX Packet header plus the skb data in the Work Request.
 859         */
 860        if (is_eth_imm(skb))
 861                return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt),
 862                                    sizeof(__be64));
 863
 864        /*
 865         * Otherwise, we're going to have to construct a Scatter gather list
 866         * of the skb body and fragments.  We also include the flits necessary
 867         * for the TX Packet Work Request and CPL.  We always have a firmware
 868         * Write Header (incorporated as part of the cpl_tx_pkt_lso and
 869         * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
 870         * message or, if we're doing a Large Send Offload, an LSO CPL message
 871         * with an embedded TX Packet Write CPL message.
 872         */
 873        flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
 874        if (skb_shinfo(skb)->gso_size)
 875                flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
 876                          sizeof(struct cpl_tx_pkt_lso_core) +
 877                          sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
 878        else
 879                flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
 880                          sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
 881        return flits;
 882}
 883
 884/**
 885 *      write_sgl - populate a Scatter/Gather List for a packet
 886 *      @skb: the packet
 887 *      @tq: the TX queue we are writing into
 888 *      @sgl: starting location for writing the SGL
 889 *      @end: points right after the end of the SGL
 890 *      @start: start offset into skb main-body data to include in the SGL
 891 *      @addr: the list of DMA bus addresses for the SGL elements
 892 *
 893 *      Generates a Scatter/Gather List for the buffers that make up a packet.
 894 *      The caller must provide adequate space for the SGL that will be written.
 895 *      The SGL includes all of the packet's page fragments and the data in its
 896 *      main body except for the first @start bytes.  @pos must be 16-byte
 897 *      aligned and within a TX descriptor with available space.  @end points
 898 *      write after the end of the SGL but does not account for any potential
 899 *      wrap around, i.e., @end > @tq->stat.
 900 */
 901static void write_sgl(const struct sk_buff *skb, struct sge_txq *tq,
 902                      struct ulptx_sgl *sgl, u64 *end, unsigned int start,
 903                      const dma_addr_t *addr)
 904{
 905        unsigned int i, len;
 906        struct ulptx_sge_pair *to;
 907        const struct skb_shared_info *si = skb_shinfo(skb);
 908        unsigned int nfrags = si->nr_frags;
 909        struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1];
 910
 911        len = skb_headlen(skb) - start;
 912        if (likely(len)) {
 913                sgl->len0 = htonl(len);
 914                sgl->addr0 = cpu_to_be64(addr[0] + start);
 915                nfrags++;
 916        } else {
 917                sgl->len0 = htonl(skb_frag_size(&si->frags[0]));
 918                sgl->addr0 = cpu_to_be64(addr[1]);
 919        }
 920
 921        sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
 922                              ULPTX_NSGE_V(nfrags));
 923        if (likely(--nfrags == 0))
 924                return;
 925        /*
 926         * Most of the complexity below deals with the possibility we hit the
 927         * end of the queue in the middle of writing the SGL.  For this case
 928         * only we create the SGL in a temporary buffer and then copy it.
 929         */
 930        to = (u8 *)end > (u8 *)tq->stat ? buf : sgl->sge;
 931
 932        for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) {
 933                to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
 934                to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i]));
 935                to->addr[0] = cpu_to_be64(addr[i]);
 936                to->addr[1] = cpu_to_be64(addr[++i]);
 937        }
 938        if (nfrags) {
 939                to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
 940                to->len[1] = cpu_to_be32(0);
 941                to->addr[0] = cpu_to_be64(addr[i + 1]);
 942        }
 943        if (unlikely((u8 *)end > (u8 *)tq->stat)) {
 944                unsigned int part0 = (u8 *)tq->stat - (u8 *)sgl->sge, part1;
 945
 946                if (likely(part0))
 947                        memcpy(sgl->sge, buf, part0);
 948                part1 = (u8 *)end - (u8 *)tq->stat;
 949                memcpy(tq->desc, (u8 *)buf + part0, part1);
 950                end = (void *)tq->desc + part1;
 951        }
 952        if ((uintptr_t)end & 8)           /* 0-pad to multiple of 16 */
 953                *end = 0;
 954}
 955
 956/**
 957 *      check_ring_tx_db - check and potentially ring a TX queue's doorbell
 958 *      @adapter: the adapter
 959 *      @tq: the TX queue
 960 *      @n: number of new descriptors to give to HW
 961 *
 962 *      Ring the doorbel for a TX queue.
 963 */
 964static inline void ring_tx_db(struct adapter *adapter, struct sge_txq *tq,
 965                              int n)
 966{
 967        /* Make sure that all writes to the TX Descriptors are committed
 968         * before we tell the hardware about them.
 969         */
 970        wmb();
 971
 972        /* If we don't have access to the new User Doorbell (T5+), use the old
 973         * doorbell mechanism; otherwise use the new BAR2 mechanism.
 974         */
 975        if (unlikely(tq->bar2_addr == NULL)) {
 976                u32 val = PIDX_V(n);
 977
 978                t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL,
 979                             QID_V(tq->cntxt_id) | val);
 980        } else {
 981                u32 val = PIDX_T5_V(n);
 982
 983                /* T4 and later chips share the same PIDX field offset within
 984                 * the doorbell, but T5 and later shrank the field in order to
 985                 * gain a bit for Doorbell Priority.  The field was absurdly
 986                 * large in the first place (14 bits) so we just use the T5
 987                 * and later limits and warn if a Queue ID is too large.
 988                 */
 989                WARN_ON(val & DBPRIO_F);
 990
 991                /* If we're only writing a single Egress Unit and the BAR2
 992                 * Queue ID is 0, we can use the Write Combining Doorbell
 993                 * Gather Buffer; otherwise we use the simple doorbell.
 994                 */
 995                if (n == 1 && tq->bar2_qid == 0) {
 996                        unsigned int index = (tq->pidx
 997                                              ? (tq->pidx - 1)
 998                                              : (tq->size - 1));
 999                        __be64 *src = (__be64 *)&tq->desc[index];
1000                        __be64 __iomem *dst = (__be64 __iomem *)(tq->bar2_addr +
1001                                                         SGE_UDB_WCDOORBELL);
1002                        unsigned int count = EQ_UNIT / sizeof(__be64);
1003
1004                        /* Copy the TX Descriptor in a tight loop in order to
1005                         * try to get it to the adapter in a single Write
1006                         * Combined transfer on the PCI-E Bus.  If the Write
1007                         * Combine fails (say because of an interrupt, etc.)
1008                         * the hardware will simply take the last write as a
1009                         * simple doorbell write with a PIDX Increment of 1
1010                         * and will fetch the TX Descriptor from memory via
1011                         * DMA.
1012                         */
1013                        while (count) {
1014                                /* the (__force u64) is because the compiler
1015                                 * doesn't understand the endian swizzling
1016                                 * going on
1017                                 */
1018                                writeq((__force u64)*src, dst);
1019                                src++;
1020                                dst++;
1021                                count--;
1022                        }
1023                } else
1024                        writel(val | QID_V(tq->bar2_qid),
1025                               tq->bar2_addr + SGE_UDB_KDOORBELL);
1026
1027                /* This Write Memory Barrier will force the write to the User
1028                 * Doorbell area to be flushed.  This is needed to prevent
1029                 * writes on different CPUs for the same queue from hitting
1030                 * the adapter out of order.  This is required when some Work
1031                 * Requests take the Write Combine Gather Buffer path (user
1032                 * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
1033                 * take the traditional path where we simply increment the
1034                 * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
1035                 * hardware DMA read the actual Work Request.
1036                 */
1037                wmb();
1038        }
1039}
1040
1041/**
1042 *      inline_tx_skb - inline a packet's data into TX descriptors
1043 *      @skb: the packet
1044 *      @tq: the TX queue where the packet will be inlined
1045 *      @pos: starting position in the TX queue to inline the packet
1046 *
1047 *      Inline a packet's contents directly into TX descriptors, starting at
1048 *      the given position within the TX DMA ring.
1049 *      Most of the complexity of this operation is dealing with wrap arounds
1050 *      in the middle of the packet we want to inline.
1051 */
1052static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *tq,
1053                          void *pos)
1054{
1055        u64 *p;
1056        int left = (void *)tq->stat - pos;
1057
1058        if (likely(skb->len <= left)) {
1059                if (likely(!skb->data_len))
1060                        skb_copy_from_linear_data(skb, pos, skb->len);
1061                else
1062                        skb_copy_bits(skb, 0, pos, skb->len);
1063                pos += skb->len;
1064        } else {
1065                skb_copy_bits(skb, 0, pos, left);
1066                skb_copy_bits(skb, left, tq->desc, skb->len - left);
1067                pos = (void *)tq->desc + (skb->len - left);
1068        }
1069
1070        /* 0-pad to multiple of 16 */
1071        p = PTR_ALIGN(pos, 8);
1072        if ((uintptr_t)p & 8)
1073                *p = 0;
1074}
1075
1076/*
1077 * Figure out what HW csum a packet wants and return the appropriate control
1078 * bits.
1079 */
1080static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
1081{
1082        int csum_type;
1083        const struct iphdr *iph = ip_hdr(skb);
1084
1085        if (iph->version == 4) {
1086                if (iph->protocol == IPPROTO_TCP)
1087                        csum_type = TX_CSUM_TCPIP;
1088                else if (iph->protocol == IPPROTO_UDP)
1089                        csum_type = TX_CSUM_UDPIP;
1090                else {
1091nocsum:
1092                        /*
1093                         * unknown protocol, disable HW csum
1094                         * and hope a bad packet is detected
1095                         */
1096                        return TXPKT_L4CSUM_DIS_F;
1097                }
1098        } else {
1099                /*
1100                 * this doesn't work with extension headers
1101                 */
1102                const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph;
1103
1104                if (ip6h->nexthdr == IPPROTO_TCP)
1105                        csum_type = TX_CSUM_TCPIP6;
1106                else if (ip6h->nexthdr == IPPROTO_UDP)
1107                        csum_type = TX_CSUM_UDPIP6;
1108                else
1109                        goto nocsum;
1110        }
1111
1112        if (likely(csum_type >= TX_CSUM_TCPIP)) {
1113                u64 hdr_len = TXPKT_IPHDR_LEN_V(skb_network_header_len(skb));
1114                int eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;
1115
1116                if (chip <= CHELSIO_T5)
1117                        hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1118                else
1119                        hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1120                return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len;
1121        } else {
1122                int start = skb_transport_offset(skb);
1123
1124                return TXPKT_CSUM_TYPE_V(csum_type) |
1125                        TXPKT_CSUM_START_V(start) |
1126                        TXPKT_CSUM_LOC_V(start + skb->csum_offset);
1127        }
1128}
1129
1130/*
1131 * Stop an Ethernet TX queue and record that state change.
1132 */
1133static void txq_stop(struct sge_eth_txq *txq)
1134{
1135        netif_tx_stop_queue(txq->txq);
1136        txq->q.stops++;
1137}
1138
1139/*
1140 * Advance our software state for a TX queue by adding n in use descriptors.
1141 */
1142static inline void txq_advance(struct sge_txq *tq, unsigned int n)
1143{
1144        tq->in_use += n;
1145        tq->pidx += n;
1146        if (tq->pidx >= tq->size)
1147                tq->pidx -= tq->size;
1148}
1149
1150/**
1151 *      t4vf_eth_xmit - add a packet to an Ethernet TX queue
1152 *      @skb: the packet
1153 *      @dev: the egress net device
1154 *
1155 *      Add a packet to an SGE Ethernet TX queue.  Runs with softirqs disabled.
1156 */
1157netdev_tx_t t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1158{
1159        u32 wr_mid;
1160        u64 cntrl, *end;
1161        int qidx, credits, max_pkt_len;
1162        unsigned int flits, ndesc;
1163        struct adapter *adapter;
1164        struct sge_eth_txq *txq;
1165        const struct port_info *pi;
1166        struct fw_eth_tx_pkt_vm_wr *wr;
1167        struct cpl_tx_pkt_core *cpl;
1168        const struct skb_shared_info *ssi;
1169        dma_addr_t addr[MAX_SKB_FRAGS + 1];
1170        const size_t fw_hdr_copy_len = (sizeof(wr->ethmacdst) +
1171                                        sizeof(wr->ethmacsrc) +
1172                                        sizeof(wr->ethtype) +
1173                                        sizeof(wr->vlantci));
1174
1175        /*
1176         * The chip minimum packet length is 10 octets but the firmware
1177         * command that we are using requires that we copy the Ethernet header
1178         * (including the VLAN tag) into the header so we reject anything
1179         * smaller than that ...
1180         */
1181        if (unlikely(skb->len < fw_hdr_copy_len))
1182                goto out_free;
1183
1184        /* Discard the packet if the length is greater than mtu */
1185        max_pkt_len = ETH_HLEN + dev->mtu;
1186        if (skb_vlan_tagged(skb))
1187                max_pkt_len += VLAN_HLEN;
1188        if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1189                goto out_free;
1190
1191        /*
1192         * Figure out which TX Queue we're going to use.
1193         */
1194        pi = netdev_priv(dev);
1195        adapter = pi->adapter;
1196        qidx = skb_get_queue_mapping(skb);
1197        BUG_ON(qidx >= pi->nqsets);
1198        txq = &adapter->sge.ethtxq[pi->first_qset + qidx];
1199
1200        if (pi->vlan_id && !skb_vlan_tag_present(skb))
1201                __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
1202                                       pi->vlan_id);
1203
1204        /*
1205         * Take this opportunity to reclaim any TX Descriptors whose DMA
1206         * transfers have completed.
1207         */
1208        reclaim_completed_tx(adapter, &txq->q, true);
1209
1210        /*
1211         * Calculate the number of flits and TX Descriptors we're going to
1212         * need along with how many TX Descriptors will be left over after
1213         * we inject our Work Request.
1214         */
1215        flits = calc_tx_flits(skb);
1216        ndesc = flits_to_desc(flits);
1217        credits = txq_avail(&txq->q) - ndesc;
1218
1219        if (unlikely(credits < 0)) {
1220                /*
1221                 * Not enough room for this packet's Work Request.  Stop the
1222                 * TX Queue and return a "busy" condition.  The queue will get
1223                 * started later on when the firmware informs us that space
1224                 * has opened up.
1225                 */
1226                txq_stop(txq);
1227                dev_err(adapter->pdev_dev,
1228                        "%s: TX ring %u full while queue awake!\n",
1229                        dev->name, qidx);
1230                return NETDEV_TX_BUSY;
1231        }
1232
1233        if (!is_eth_imm(skb) &&
1234            unlikely(map_skb(adapter->pdev_dev, skb, addr) < 0)) {
1235                /*
1236                 * We need to map the skb into PCI DMA space (because it can't
1237                 * be in-lined directly into the Work Request) and the mapping
1238                 * operation failed.  Record the error and drop the packet.
1239                 */
1240                txq->mapping_err++;
1241                goto out_free;
1242        }
1243
1244        wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1245        if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1246                /*
1247                 * After we're done injecting the Work Request for this
1248                 * packet, we'll be below our "stop threshold" so stop the TX
1249                 * Queue now and schedule a request for an SGE Egress Queue
1250                 * Update message.  The queue will get started later on when
1251                 * the firmware processes this Work Request and sends us an
1252                 * Egress Queue Status Update message indicating that space
1253                 * has opened up.
1254                 */
1255                txq_stop(txq);
1256                wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1257        }
1258
1259        /*
1260         * Start filling in our Work Request.  Note that we do _not_ handle
1261         * the WR Header wrapping around the TX Descriptor Ring.  If our
1262         * maximum header size ever exceeds one TX Descriptor, we'll need to
1263         * do something else here.
1264         */
1265        BUG_ON(DIV_ROUND_UP(ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1);
1266        wr = (void *)&txq->q.desc[txq->q.pidx];
1267        wr->equiq_to_len16 = cpu_to_be32(wr_mid);
1268        wr->r3[0] = cpu_to_be32(0);
1269        wr->r3[1] = cpu_to_be32(0);
1270        skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len);
1271        end = (u64 *)wr + flits;
1272
1273        /*
1274         * If this is a Large Send Offload packet we'll put in an LSO CPL
1275         * message with an encapsulated TX Packet CPL message.  Otherwise we
1276         * just use a TX Packet CPL message.
1277         */
1278        ssi = skb_shinfo(skb);
1279        if (ssi->gso_size) {
1280                struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1281                bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1282                int l3hdr_len = skb_network_header_len(skb);
1283                int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1284
1285                wr->op_immdlen =
1286                        cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1287                                    FW_WR_IMMDLEN_V(sizeof(*lso) +
1288                                                    sizeof(*cpl)));
1289                /*
1290                 * Fill in the LSO CPL message.
1291                 */
1292                lso->lso_ctrl =
1293                        cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1294                                    LSO_FIRST_SLICE_F |
1295                                    LSO_LAST_SLICE_F |
1296                                    LSO_IPV6_V(v6) |
1297                                    LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1298                                    LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1299                                    LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1300                lso->ipid_ofst = cpu_to_be16(0);
1301                lso->mss = cpu_to_be16(ssi->gso_size);
1302                lso->seqno_offset = cpu_to_be32(0);
1303                if (is_t4(adapter->params.chip))
1304                        lso->len = cpu_to_be32(skb->len);
1305                else
1306                        lso->len = cpu_to_be32(LSO_T5_XFER_SIZE_V(skb->len));
1307
1308                /*
1309                 * Set up TX Packet CPL pointer, control word and perform
1310                 * accounting.
1311                 */
1312                cpl = (void *)(lso + 1);
1313
1314                if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
1315                        cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1316                else
1317                        cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1318
1319                cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1320                                           TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1321                         TXPKT_IPHDR_LEN_V(l3hdr_len);
1322                txq->tso++;
1323                txq->tx_cso += ssi->gso_segs;
1324        } else {
1325                int len;
1326
1327                len = is_eth_imm(skb) ? skb->len + sizeof(*cpl) : sizeof(*cpl);
1328                wr->op_immdlen =
1329                        cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1330                                    FW_WR_IMMDLEN_V(len));
1331
1332                /*
1333                 * Set up TX Packet CPL pointer, control word and perform
1334                 * accounting.
1335                 */
1336                cpl = (void *)(wr + 1);
1337                if (skb->ip_summed == CHECKSUM_PARTIAL) {
1338                        cntrl = hwcsum(adapter->params.chip, skb) |
1339                                TXPKT_IPCSUM_DIS_F;
1340                        txq->tx_cso++;
1341                } else
1342                        cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1343        }
1344
1345        /*
1346         * If there's a VLAN tag present, add that to the list of things to
1347         * do in this Work Request.
1348         */
1349        if (skb_vlan_tag_present(skb)) {
1350                txq->vlan_ins++;
1351                cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1352        }
1353
1354        /*
1355         * Fill in the TX Packet CPL message header.
1356         */
1357        cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) |
1358                                 TXPKT_INTF_V(pi->port_id) |
1359                                 TXPKT_PF_V(0));
1360        cpl->pack = cpu_to_be16(0);
1361        cpl->len = cpu_to_be16(skb->len);
1362        cpl->ctrl1 = cpu_to_be64(cntrl);
1363
1364#ifdef T4_TRACE
1365        T4_TRACE5(adapter->tb[txq->q.cntxt_id & 7],
1366                  "eth_xmit: ndesc %u, credits %u, pidx %u, len %u, frags %u",
1367                  ndesc, credits, txq->q.pidx, skb->len, ssi->nr_frags);
1368#endif
1369
1370        /*
1371         * Fill in the body of the TX Packet CPL message with either in-lined
1372         * data or a Scatter/Gather List.
1373         */
1374        if (is_eth_imm(skb)) {
1375                /*
1376                 * In-line the packet's data and free the skb since we don't
1377                 * need it any longer.
1378                 */
1379                inline_tx_skb(skb, &txq->q, cpl + 1);
1380                dev_consume_skb_any(skb);
1381        } else {
1382                /*
1383                 * Write the skb's Scatter/Gather list into the TX Packet CPL
1384                 * message and retain a pointer to the skb so we can free it
1385                 * later when its DMA completes.  (We store the skb pointer
1386                 * in the Software Descriptor corresponding to the last TX
1387                 * Descriptor used by the Work Request.)
1388                 *
1389                 * The retained skb will be freed when the corresponding TX
1390                 * Descriptors are reclaimed after their DMAs complete.
1391                 * However, this could take quite a while since, in general,
1392                 * the hardware is set up to be lazy about sending DMA
1393                 * completion notifications to us and we mostly perform TX
1394                 * reclaims in the transmit routine.
1395                 *
1396                 * This is good for performamce but means that we rely on new
1397                 * TX packets arriving to run the destructors of completed
1398                 * packets, which open up space in their sockets' send queues.
1399                 * Sometimes we do not get such new packets causing TX to
1400                 * stall.  A single UDP transmitter is a good example of this
1401                 * situation.  We have a clean up timer that periodically
1402                 * reclaims completed packets but it doesn't run often enough
1403                 * (nor do we want it to) to prevent lengthy stalls.  A
1404                 * solution to this problem is to run the destructor early,
1405                 * after the packet is queued but before it's DMAd.  A con is
1406                 * that we lie to socket memory accounting, but the amount of
1407                 * extra memory is reasonable (limited by the number of TX
1408                 * descriptors), the packets do actually get freed quickly by
1409                 * new packets almost always, and for protocols like TCP that
1410                 * wait for acks to really free up the data the extra memory
1411                 * is even less.  On the positive side we run the destructors
1412                 * on the sending CPU rather than on a potentially different
1413                 * completing CPU, usually a good thing.
1414                 *
1415                 * Run the destructor before telling the DMA engine about the
1416                 * packet to make sure it doesn't complete and get freed
1417                 * prematurely.
1418                 */
1419                struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1);
1420                struct sge_txq *tq = &txq->q;
1421                int last_desc;
1422
1423                /*
1424                 * If the Work Request header was an exact multiple of our TX
1425                 * Descriptor length, then it's possible that the starting SGL
1426                 * pointer lines up exactly with the end of our TX Descriptor
1427                 * ring.  If that's the case, wrap around to the beginning
1428                 * here ...
1429                 */
1430                if (unlikely((void *)sgl == (void *)tq->stat)) {
1431                        sgl = (void *)tq->desc;
1432                        end = ((void *)tq->desc + ((void *)end - (void *)tq->stat));
1433                }
1434
1435                write_sgl(skb, tq, sgl, end, 0, addr);
1436                skb_orphan(skb);
1437
1438                last_desc = tq->pidx + ndesc - 1;
1439                if (last_desc >= tq->size)
1440                        last_desc -= tq->size;
1441                tq->sdesc[last_desc].skb = skb;
1442                tq->sdesc[last_desc].sgl = sgl;
1443        }
1444
1445        /*
1446         * Advance our internal TX Queue state, tell the hardware about
1447         * the new TX descriptors and return success.
1448         */
1449        txq_advance(&txq->q, ndesc);
1450        netif_trans_update(dev);
1451        ring_tx_db(adapter, &txq->q, ndesc);
1452        return NETDEV_TX_OK;
1453
1454out_free:
1455        /*
1456         * An error of some sort happened.  Free the TX skb and tell the
1457         * OS that we've "dealt" with the packet ...
1458         */
1459        dev_kfree_skb_any(skb);
1460        return NETDEV_TX_OK;
1461}
1462
1463/**
1464 *      copy_frags - copy fragments from gather list into skb_shared_info
1465 *      @skb: destination skb
1466 *      @gl: source internal packet gather list
1467 *      @offset: packet start offset in first page
1468 *
1469 *      Copy an internal packet gather list into a Linux skb_shared_info
1470 *      structure.
1471 */
1472static inline void copy_frags(struct sk_buff *skb,
1473                              const struct pkt_gl *gl,
1474                              unsigned int offset)
1475{
1476        int i;
1477
1478        /* usually there's just one frag */
1479        __skb_fill_page_desc(skb, 0, gl->frags[0].page,
1480                             gl->frags[0].offset + offset,
1481                             gl->frags[0].size - offset);
1482        skb_shinfo(skb)->nr_frags = gl->nfrags;
1483        for (i = 1; i < gl->nfrags; i++)
1484                __skb_fill_page_desc(skb, i, gl->frags[i].page,
1485                                     gl->frags[i].offset,
1486                                     gl->frags[i].size);
1487
1488        /* get a reference to the last page, we don't own it */
1489        get_page(gl->frags[gl->nfrags - 1].page);
1490}
1491
1492/**
1493 *      t4vf_pktgl_to_skb - build an sk_buff from a packet gather list
1494 *      @gl: the gather list
1495 *      @skb_len: size of sk_buff main body if it carries fragments
1496 *      @pull_len: amount of data to move to the sk_buff's main body
1497 *
1498 *      Builds an sk_buff from the given packet gather list.  Returns the
1499 *      sk_buff or %NULL if sk_buff allocation failed.
1500 */
1501static struct sk_buff *t4vf_pktgl_to_skb(const struct pkt_gl *gl,
1502                                         unsigned int skb_len,
1503                                         unsigned int pull_len)
1504{
1505        struct sk_buff *skb;
1506
1507        /*
1508         * If the ingress packet is small enough, allocate an skb large enough
1509         * for all of the data and copy it inline.  Otherwise, allocate an skb
1510         * with enough room to pull in the header and reference the rest of
1511         * the data via the skb fragment list.
1512         *
1513         * Below we rely on RX_COPY_THRES being less than the smallest Rx
1514         * buff!  size, which is expected since buffers are at least
1515         * PAGE_SIZEd.  In this case packets up to RX_COPY_THRES have only one
1516         * fragment.
1517         */
1518        if (gl->tot_len <= RX_COPY_THRES) {
1519                /* small packets have only one fragment */
1520                skb = alloc_skb(gl->tot_len, GFP_ATOMIC);
1521                if (unlikely(!skb))
1522                        goto out;
1523                __skb_put(skb, gl->tot_len);
1524                skb_copy_to_linear_data(skb, gl->va, gl->tot_len);
1525        } else {
1526                skb = alloc_skb(skb_len, GFP_ATOMIC);
1527                if (unlikely(!skb))
1528                        goto out;
1529                __skb_put(skb, pull_len);
1530                skb_copy_to_linear_data(skb, gl->va, pull_len);
1531
1532                copy_frags(skb, gl, pull_len);
1533                skb->len = gl->tot_len;
1534                skb->data_len = skb->len - pull_len;
1535                skb->truesize += skb->data_len;
1536        }
1537
1538out:
1539        return skb;
1540}
1541
1542/**
1543 *      t4vf_pktgl_free - free a packet gather list
1544 *      @gl: the gather list
1545 *
1546 *      Releases the pages of a packet gather list.  We do not own the last
1547 *      page on the list and do not free it.
1548 */
1549static void t4vf_pktgl_free(const struct pkt_gl *gl)
1550{
1551        int frag;
1552
1553        frag = gl->nfrags - 1;
1554        while (frag--)
1555                put_page(gl->frags[frag].page);
1556}
1557
1558/**
1559 *      do_gro - perform Generic Receive Offload ingress packet processing
1560 *      @rxq: ingress RX Ethernet Queue
1561 *      @gl: gather list for ingress packet
1562 *      @pkt: CPL header for last packet fragment
1563 *
1564 *      Perform Generic Receive Offload (GRO) ingress packet processing.
1565 *      We use the standard Linux GRO interfaces for this.
1566 */
1567static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
1568                   const struct cpl_rx_pkt *pkt)
1569{
1570        struct adapter *adapter = rxq->rspq.adapter;
1571        struct sge *s = &adapter->sge;
1572        struct port_info *pi;
1573        int ret;
1574        struct sk_buff *skb;
1575
1576        skb = napi_get_frags(&rxq->rspq.napi);
1577        if (unlikely(!skb)) {
1578                t4vf_pktgl_free(gl);
1579                rxq->stats.rx_drops++;
1580                return;
1581        }
1582
1583        copy_frags(skb, gl, s->pktshift);
1584        skb->len = gl->tot_len - s->pktshift;
1585        skb->data_len = skb->len;
1586        skb->truesize += skb->data_len;
1587        skb->ip_summed = CHECKSUM_UNNECESSARY;
1588        skb_record_rx_queue(skb, rxq->rspq.idx);
1589        pi = netdev_priv(skb->dev);
1590
1591        if (pkt->vlan_ex && !pi->vlan_id) {
1592                __vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
1593                                        be16_to_cpu(pkt->vlan));
1594                rxq->stats.vlan_ex++;
1595        }
1596        ret = napi_gro_frags(&rxq->rspq.napi);
1597
1598        if (ret == GRO_HELD)
1599                rxq->stats.lro_pkts++;
1600        else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE)
1601                rxq->stats.lro_merged++;
1602        rxq->stats.pkts++;
1603        rxq->stats.rx_cso++;
1604}
1605
1606/**
1607 *      t4vf_ethrx_handler - process an ingress ethernet packet
1608 *      @rspq: the response queue that received the packet
1609 *      @rsp: the response queue descriptor holding the RX_PKT message
1610 *      @gl: the gather list of packet fragments
1611 *
1612 *      Process an ingress ethernet packet and deliver it to the stack.
1613 */
1614int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp,
1615                       const struct pkt_gl *gl)
1616{
1617        struct sk_buff *skb;
1618        const struct cpl_rx_pkt *pkt = (void *)rsp;
1619        bool csum_ok = pkt->csum_calc && !pkt->err_vec &&
1620                       (rspq->netdev->features & NETIF_F_RXCSUM);
1621        struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq);
1622        struct adapter *adapter = rspq->adapter;
1623        struct sge *s = &adapter->sge;
1624        struct port_info *pi;
1625
1626        /*
1627         * If this is a good TCP packet and we have Generic Receive Offload
1628         * enabled, handle the packet in the GRO path.
1629         */
1630        if ((pkt->l2info & cpu_to_be32(RXF_TCP_F)) &&
1631            (rspq->netdev->features & NETIF_F_GRO) && csum_ok &&
1632            !pkt->ip_frag) {
1633                do_gro(rxq, gl, pkt);
1634                return 0;
1635        }
1636
1637        /*
1638         * Convert the Packet Gather List into an skb.
1639         */
1640        skb = t4vf_pktgl_to_skb(gl, RX_SKB_LEN, RX_PULL_LEN);
1641        if (unlikely(!skb)) {
1642                t4vf_pktgl_free(gl);
1643                rxq->stats.rx_drops++;
1644                return 0;
1645        }
1646        __skb_pull(skb, s->pktshift);
1647        skb->protocol = eth_type_trans(skb, rspq->netdev);
1648        skb_record_rx_queue(skb, rspq->idx);
1649        pi = netdev_priv(skb->dev);
1650        rxq->stats.pkts++;
1651
1652        if (csum_ok && !pkt->err_vec &&
1653            (be32_to_cpu(pkt->l2info) & (RXF_UDP_F | RXF_TCP_F))) {
1654                if (!pkt->ip_frag) {
1655                        skb->ip_summed = CHECKSUM_UNNECESSARY;
1656                        rxq->stats.rx_cso++;
1657                } else if (pkt->l2info & htonl(RXF_IP_F)) {
1658                        __sum16 c = (__force __sum16)pkt->csum;
1659                        skb->csum = csum_unfold(c);
1660                        skb->ip_summed = CHECKSUM_COMPLETE;
1661                        rxq->stats.rx_cso++;
1662                }
1663        } else
1664                skb_checksum_none_assert(skb);
1665
1666        if (pkt->vlan_ex && !pi->vlan_id) {
1667                rxq->stats.vlan_ex++;
1668                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1669                                       be16_to_cpu(pkt->vlan));
1670        }
1671
1672        netif_receive_skb(skb);
1673
1674        return 0;
1675}
1676
1677/**
1678 *      is_new_response - check if a response is newly written
1679 *      @rc: the response control descriptor
1680 *      @rspq: the response queue
1681 *
1682 *      Returns true if a response descriptor contains a yet unprocessed
1683 *      response.
1684 */
1685static inline bool is_new_response(const struct rsp_ctrl *rc,
1686                                   const struct sge_rspq *rspq)
1687{
1688        return ((rc->type_gen >> RSPD_GEN_S) & 0x1) == rspq->gen;
1689}
1690
1691/**
1692 *      restore_rx_bufs - put back a packet's RX buffers
1693 *      @gl: the packet gather list
1694 *      @fl: the SGE Free List
1695 *      @frags: how many fragments in @si
1696 *
1697 *      Called when we find out that the current packet, @si, can't be
1698 *      processed right away for some reason.  This is a very rare event and
1699 *      there's no effort to make this suspension/resumption process
1700 *      particularly efficient.
1701 *
1702 *      We implement the suspension by putting all of the RX buffers associated
1703 *      with the current packet back on the original Free List.  The buffers
1704 *      have already been unmapped and are left unmapped, we mark them as
1705 *      unmapped in order to prevent further unmapping attempts.  (Effectively
1706 *      this function undoes the series of @unmap_rx_buf calls which were done
1707 *      to create the current packet's gather list.)  This leaves us ready to
1708 *      restart processing of the packet the next time we start processing the
1709 *      RX Queue ...
1710 */
1711static void restore_rx_bufs(const struct pkt_gl *gl, struct sge_fl *fl,
1712                            int frags)
1713{
1714        struct rx_sw_desc *sdesc;
1715
1716        while (frags--) {
1717                if (fl->cidx == 0)
1718                        fl->cidx = fl->size - 1;
1719                else
1720                        fl->cidx--;
1721                sdesc = &fl->sdesc[fl->cidx];
1722                sdesc->page = gl->frags[frags].page;
1723                sdesc->dma_addr |= RX_UNMAPPED_BUF;
1724                fl->avail++;
1725        }
1726}
1727
1728/**
1729 *      rspq_next - advance to the next entry in a response queue
1730 *      @rspq: the queue
1731 *
1732 *      Updates the state of a response queue to advance it to the next entry.
1733 */
1734static inline void rspq_next(struct sge_rspq *rspq)
1735{
1736        rspq->cur_desc = (void *)rspq->cur_desc + rspq->iqe_len;
1737        if (unlikely(++rspq->cidx == rspq->size)) {
1738                rspq->cidx = 0;
1739                rspq->gen ^= 1;
1740                rspq->cur_desc = rspq->desc;
1741        }
1742}
1743
1744/**
1745 *      process_responses - process responses from an SGE response queue
1746 *      @rspq: the ingress response queue to process
1747 *      @budget: how many responses can be processed in this round
1748 *
1749 *      Process responses from a Scatter Gather Engine response queue up to
1750 *      the supplied budget.  Responses include received packets as well as
1751 *      control messages from firmware or hardware.
1752 *
1753 *      Additionally choose the interrupt holdoff time for the next interrupt
1754 *      on this queue.  If the system is under memory shortage use a fairly
1755 *      long delay to help recovery.
1756 */
1757static int process_responses(struct sge_rspq *rspq, int budget)
1758{
1759        struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq);
1760        struct adapter *adapter = rspq->adapter;
1761        struct sge *s = &adapter->sge;
1762        int budget_left = budget;
1763
1764        while (likely(budget_left)) {
1765                int ret, rsp_type;
1766                const struct rsp_ctrl *rc;
1767
1768                rc = (void *)rspq->cur_desc + (rspq->iqe_len - sizeof(*rc));
1769                if (!is_new_response(rc, rspq))
1770                        break;
1771
1772                /*
1773                 * Figure out what kind of response we've received from the
1774                 * SGE.
1775                 */
1776                dma_rmb();
1777                rsp_type = RSPD_TYPE_G(rc->type_gen);
1778                if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) {
1779                        struct page_frag *fp;
1780                        struct pkt_gl gl;
1781                        const struct rx_sw_desc *sdesc;
1782                        u32 bufsz, frag;
1783                        u32 len = be32_to_cpu(rc->pldbuflen_qid);
1784
1785                        /*
1786                         * If we get a "new buffer" message from the SGE we
1787                         * need to move on to the next Free List buffer.
1788                         */
1789                        if (len & RSPD_NEWBUF_F) {
1790                                /*
1791                                 * We get one "new buffer" message when we
1792                                 * first start up a queue so we need to ignore
1793                                 * it when our offset into the buffer is 0.
1794                                 */
1795                                if (likely(rspq->offset > 0)) {
1796                                        free_rx_bufs(rspq->adapter, &rxq->fl,
1797                                                     1);
1798                                        rspq->offset = 0;
1799                                }
1800                                len = RSPD_LEN_G(len);
1801                        }
1802                        gl.tot_len = len;
1803
1804                        /*
1805                         * Gather packet fragments.
1806                         */
1807                        for (frag = 0, fp = gl.frags; /**/; frag++, fp++) {
1808                                BUG_ON(frag >= MAX_SKB_FRAGS);
1809                                BUG_ON(rxq->fl.avail == 0);
1810                                sdesc = &rxq->fl.sdesc[rxq->fl.cidx];
1811                                bufsz = get_buf_size(adapter, sdesc);
1812                                fp->page = sdesc->page;
1813                                fp->offset = rspq->offset;
1814                                fp->size = min(bufsz, len);
1815                                len -= fp->size;
1816                                if (!len)
1817                                        break;
1818                                unmap_rx_buf(rspq->adapter, &rxq->fl);
1819                        }
1820                        gl.nfrags = frag+1;
1821
1822                        /*
1823                         * Last buffer remains mapped so explicitly make it
1824                         * coherent for CPU access and start preloading first
1825                         * cache line ...
1826                         */
1827                        dma_sync_single_for_cpu(rspq->adapter->pdev_dev,
1828                                                get_buf_addr(sdesc),
1829                                                fp->size, DMA_FROM_DEVICE);
1830                        gl.va = (page_address(gl.frags[0].page) +
1831                                 gl.frags[0].offset);
1832                        prefetch(gl.va);
1833
1834                        /*
1835                         * Hand the new ingress packet to the handler for
1836                         * this Response Queue.
1837                         */
1838                        ret = rspq->handler(rspq, rspq->cur_desc, &gl);
1839                        if (likely(ret == 0))
1840                                rspq->offset += ALIGN(fp->size, s->fl_align);
1841                        else
1842                                restore_rx_bufs(&gl, &rxq->fl, frag);
1843                } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) {
1844                        ret = rspq->handler(rspq, rspq->cur_desc, NULL);
1845                } else {
1846                        WARN_ON(rsp_type > RSPD_TYPE_CPL_X);
1847                        ret = 0;
1848                }
1849
1850                if (unlikely(ret)) {
1851                        /*
1852                         * Couldn't process descriptor, back off for recovery.
1853                         * We use the SGE's last timer which has the longest
1854                         * interrupt coalescing value ...
1855                         */
1856                        const int NOMEM_TIMER_IDX = SGE_NTIMERS-1;
1857                        rspq->next_intr_params =
1858                                QINTR_TIMER_IDX_V(NOMEM_TIMER_IDX);
1859                        break;
1860                }
1861
1862                rspq_next(rspq);
1863                budget_left--;
1864        }
1865
1866        /*
1867         * If this is a Response Queue with an associated Free List and
1868         * at least two Egress Queue units available in the Free List
1869         * for new buffer pointers, refill the Free List.
1870         */
1871        if (rspq->offset >= 0 &&
1872            fl_cap(&rxq->fl) - rxq->fl.avail >= 2*FL_PER_EQ_UNIT)
1873                __refill_fl(rspq->adapter, &rxq->fl);
1874        return budget - budget_left;
1875}
1876
1877/**
1878 *      napi_rx_handler - the NAPI handler for RX processing
1879 *      @napi: the napi instance
1880 *      @budget: how many packets we can process in this round
1881 *
1882 *      Handler for new data events when using NAPI.  This does not need any
1883 *      locking or protection from interrupts as data interrupts are off at
1884 *      this point and other adapter interrupts do not interfere (the latter
1885 *      in not a concern at all with MSI-X as non-data interrupts then have
1886 *      a separate handler).
1887 */
1888static int napi_rx_handler(struct napi_struct *napi, int budget)
1889{
1890        unsigned int intr_params;
1891        struct sge_rspq *rspq = container_of(napi, struct sge_rspq, napi);
1892        int work_done = process_responses(rspq, budget);
1893        u32 val;
1894
1895        if (likely(work_done < budget)) {
1896                napi_complete_done(napi, work_done);
1897                intr_params = rspq->next_intr_params;
1898                rspq->next_intr_params = rspq->intr_params;
1899        } else
1900                intr_params = QINTR_TIMER_IDX_V(SGE_TIMER_UPD_CIDX);
1901
1902        if (unlikely(work_done == 0))
1903                rspq->unhandled_irqs++;
1904
1905        val = CIDXINC_V(work_done) | SEINTARM_V(intr_params);
1906        /* If we don't have access to the new User GTS (T5+), use the old
1907         * doorbell mechanism; otherwise use the new BAR2 mechanism.
1908         */
1909        if (unlikely(!rspq->bar2_addr)) {
1910                t4_write_reg(rspq->adapter,
1911                             T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
1912                             val | INGRESSQID_V((u32)rspq->cntxt_id));
1913        } else {
1914                writel(val | INGRESSQID_V(rspq->bar2_qid),
1915                       rspq->bar2_addr + SGE_UDB_GTS);
1916                wmb();
1917        }
1918        return work_done;
1919}
1920
1921/*
1922 * The MSI-X interrupt handler for an SGE response queue for the NAPI case
1923 * (i.e., response queue serviced by NAPI polling).
1924 */
1925irqreturn_t t4vf_sge_intr_msix(int irq, void *cookie)
1926{
1927        struct sge_rspq *rspq = cookie;
1928
1929        napi_schedule(&rspq->napi);
1930        return IRQ_HANDLED;
1931}
1932
1933/*
1934 * Process the indirect interrupt entries in the interrupt queue and kick off
1935 * NAPI for each queue that has generated an entry.
1936 */
1937static unsigned int process_intrq(struct adapter *adapter)
1938{
1939        struct sge *s = &adapter->sge;
1940        struct sge_rspq *intrq = &s->intrq;
1941        unsigned int work_done;
1942        u32 val;
1943
1944        spin_lock(&adapter->sge.intrq_lock);
1945        for (work_done = 0; ; work_done++) {
1946                const struct rsp_ctrl *rc;
1947                unsigned int qid, iq_idx;
1948                struct sge_rspq *rspq;
1949
1950                /*
1951                 * Grab the next response from the interrupt queue and bail
1952                 * out if it's not a new response.
1953                 */
1954                rc = (void *)intrq->cur_desc + (intrq->iqe_len - sizeof(*rc));
1955                if (!is_new_response(rc, intrq))
1956                        break;
1957
1958                /*
1959                 * If the response isn't a forwarded interrupt message issue a
1960                 * error and go on to the next response message.  This should
1961                 * never happen ...
1962                 */
1963                dma_rmb();
1964                if (unlikely(RSPD_TYPE_G(rc->type_gen) != RSPD_TYPE_INTR_X)) {
1965                        dev_err(adapter->pdev_dev,
1966                                "Unexpected INTRQ response type %d\n",
1967                                RSPD_TYPE_G(rc->type_gen));
1968                        continue;
1969                }
1970
1971                /*
1972                 * Extract the Queue ID from the interrupt message and perform
1973                 * sanity checking to make sure it really refers to one of our
1974                 * Ingress Queues which is active and matches the queue's ID.
1975                 * None of these error conditions should ever happen so we may
1976                 * want to either make them fatal and/or conditionalized under
1977                 * DEBUG.
1978                 */
1979                qid = RSPD_QID_G(be32_to_cpu(rc->pldbuflen_qid));
1980                iq_idx = IQ_IDX(s, qid);
1981                if (unlikely(iq_idx >= MAX_INGQ)) {
1982                        dev_err(adapter->pdev_dev,
1983                                "Ingress QID %d out of range\n", qid);
1984                        continue;
1985                }
1986                rspq = s->ingr_map[iq_idx];
1987                if (unlikely(rspq == NULL)) {
1988                        dev_err(adapter->pdev_dev,
1989                                "Ingress QID %d RSPQ=NULL\n", qid);
1990                        continue;
1991                }
1992                if (unlikely(rspq->abs_id != qid)) {
1993                        dev_err(adapter->pdev_dev,
1994                                "Ingress QID %d refers to RSPQ %d\n",
1995                                qid, rspq->abs_id);
1996                        continue;
1997                }
1998
1999                /*
2000                 * Schedule NAPI processing on the indicated Response Queue
2001                 * and move on to the next entry in the Forwarded Interrupt
2002                 * Queue.
2003                 */
2004                napi_schedule(&rspq->napi);
2005                rspq_next(intrq);
2006        }
2007
2008        val = CIDXINC_V(work_done) | SEINTARM_V(intrq->intr_params);
2009        /* If we don't have access to the new User GTS (T5+), use the old
2010         * doorbell mechanism; otherwise use the new BAR2 mechanism.
2011         */
2012        if (unlikely(!intrq->bar2_addr)) {
2013                t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
2014                             val | INGRESSQID_V(intrq->cntxt_id));
2015        } else {
2016                writel(val | INGRESSQID_V(intrq->bar2_qid),
2017                       intrq->bar2_addr + SGE_UDB_GTS);
2018                wmb();
2019        }
2020
2021        spin_unlock(&adapter->sge.intrq_lock);
2022
2023        return work_done;
2024}
2025
2026/*
2027 * The MSI interrupt handler handles data events from SGE response queues as
2028 * well as error and other async events as they all use the same MSI vector.
2029 */
2030static irqreturn_t t4vf_intr_msi(int irq, void *cookie)
2031{
2032        struct adapter *adapter = cookie;
2033
2034        process_intrq(adapter);
2035        return IRQ_HANDLED;
2036}
2037
2038/**
2039 *      t4vf_intr_handler - select the top-level interrupt handler
2040 *      @adapter: the adapter
2041 *
2042 *      Selects the top-level interrupt handler based on the type of interrupts
2043 *      (MSI-X or MSI).
2044 */
2045irq_handler_t t4vf_intr_handler(struct adapter *adapter)
2046{
2047        BUG_ON((adapter->flags &
2048               (CXGB4VF_USING_MSIX | CXGB4VF_USING_MSI)) == 0);
2049        if (adapter->flags & CXGB4VF_USING_MSIX)
2050                return t4vf_sge_intr_msix;
2051        else
2052                return t4vf_intr_msi;
2053}
2054
2055/**
2056 *      sge_rx_timer_cb - perform periodic maintenance of SGE RX queues
2057 *      @t: Rx timer
2058 *
2059 *      Runs periodically from a timer to perform maintenance of SGE RX queues.
2060 *
2061 *      a) Replenishes RX queues that have run out due to memory shortage.
2062 *      Normally new RX buffers are added when existing ones are consumed but
2063 *      when out of memory a queue can become empty.  We schedule NAPI to do
2064 *      the actual refill.
2065 */
2066static void sge_rx_timer_cb(struct timer_list *t)
2067{
2068        struct adapter *adapter = from_timer(adapter, t, sge.rx_timer);
2069        struct sge *s = &adapter->sge;
2070        unsigned int i;
2071
2072        /*
2073         * Scan the "Starving Free Lists" flag array looking for any Free
2074         * Lists in need of more free buffers.  If we find one and it's not
2075         * being actively polled, then bump its "starving" counter and attempt
2076         * to refill it.  If we're successful in adding enough buffers to push
2077         * the Free List over the starving threshold, then we can clear its
2078         * "starving" status.
2079         */
2080        for (i = 0; i < ARRAY_SIZE(s->starving_fl); i++) {
2081                unsigned long m;
2082
2083                for (m = s->starving_fl[i]; m; m &= m - 1) {
2084                        unsigned int id = __ffs(m) + i * BITS_PER_LONG;
2085                        struct sge_fl *fl = s->egr_map[id];
2086
2087                        clear_bit(id, s->starving_fl);
2088                        smp_mb__after_atomic();
2089
2090                        /*
2091                         * Since we are accessing fl without a lock there's a
2092                         * small probability of a false positive where we
2093                         * schedule napi but the FL is no longer starving.
2094                         * No biggie.
2095                         */
2096                        if (fl_starving(adapter, fl)) {
2097                                struct sge_eth_rxq *rxq;
2098
2099                                rxq = container_of(fl, struct sge_eth_rxq, fl);
2100                                if (napi_reschedule(&rxq->rspq.napi))
2101                                        fl->starving++;
2102                                else
2103                                        set_bit(id, s->starving_fl);
2104                        }
2105                }
2106        }
2107
2108        /*
2109         * Reschedule the next scan for starving Free Lists ...
2110         */
2111        mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD);
2112}
2113
2114/**
2115 *      sge_tx_timer_cb - perform periodic maintenance of SGE Tx queues
2116 *      @t: Tx timer
2117 *
2118 *      Runs periodically from a timer to perform maintenance of SGE TX queues.
2119 *
2120 *      b) Reclaims completed Tx packets for the Ethernet queues.  Normally
2121 *      packets are cleaned up by new Tx packets, this timer cleans up packets
2122 *      when no new packets are being submitted.  This is essential for pktgen,
2123 *      at least.
2124 */
2125static void sge_tx_timer_cb(struct timer_list *t)
2126{
2127        struct adapter *adapter = from_timer(adapter, t, sge.tx_timer);
2128        struct sge *s = &adapter->sge;
2129        unsigned int i, budget;
2130
2131        budget = MAX_TIMER_TX_RECLAIM;
2132        i = s->ethtxq_rover;
2133        do {
2134                struct sge_eth_txq *txq = &s->ethtxq[i];
2135
2136                if (reclaimable(&txq->q) && __netif_tx_trylock(txq->txq)) {
2137                        int avail = reclaimable(&txq->q);
2138
2139                        if (avail > budget)
2140                                avail = budget;
2141
2142                        free_tx_desc(adapter, &txq->q, avail, true);
2143                        txq->q.in_use -= avail;
2144                        __netif_tx_unlock(txq->txq);
2145
2146                        budget -= avail;
2147                        if (!budget)
2148                                break;
2149                }
2150
2151                i++;
2152                if (i >= s->ethqsets)
2153                        i = 0;
2154        } while (i != s->ethtxq_rover);
2155        s->ethtxq_rover = i;
2156
2157        /*
2158         * If we found too many reclaimable packets schedule a timer in the
2159         * near future to continue where we left off.  Otherwise the next timer
2160         * will be at its normal interval.
2161         */
2162        mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2));
2163}
2164
2165/**
2166 *      bar2_address - return the BAR2 address for an SGE Queue's Registers
2167 *      @adapter: the adapter
2168 *      @qid: the SGE Queue ID
2169 *      @qtype: the SGE Queue Type (Egress or Ingress)
2170 *      @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
2171 *
2172 *      Returns the BAR2 address for the SGE Queue Registers associated with
2173 *      @qid.  If BAR2 SGE Registers aren't available, returns NULL.  Also
2174 *      returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
2175 *      Queue Registers.  If the BAR2 Queue ID is 0, then "Inferred Queue ID"
2176 *      Registers are supported (e.g. the Write Combining Doorbell Buffer).
2177 */
2178static void __iomem *bar2_address(struct adapter *adapter,
2179                                  unsigned int qid,
2180                                  enum t4_bar2_qtype qtype,
2181                                  unsigned int *pbar2_qid)
2182{
2183        u64 bar2_qoffset;
2184        int ret;
2185
2186        ret = t4vf_bar2_sge_qregs(adapter, qid, qtype,
2187                                  &bar2_qoffset, pbar2_qid);
2188        if (ret)
2189                return NULL;
2190
2191        return adapter->bar2 + bar2_qoffset;
2192}
2193
2194/**
2195 *      t4vf_sge_alloc_rxq - allocate an SGE RX Queue
2196 *      @adapter: the adapter
2197 *      @rspq: pointer to to the new rxq's Response Queue to be filled in
2198 *      @iqasynch: if 0, a normal rspq; if 1, an asynchronous event queue
2199 *      @dev: the network device associated with the new rspq
2200 *      @intr_dest: MSI-X vector index (overriden in MSI mode)
2201 *      @fl: pointer to the new rxq's Free List to be filled in
2202 *      @hnd: the interrupt handler to invoke for the rspq
2203 */
2204int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,
2205                       bool iqasynch, struct net_device *dev,
2206                       int intr_dest,
2207                       struct sge_fl *fl, rspq_handler_t hnd)
2208{
2209        struct sge *s = &adapter->sge;
2210        struct port_info *pi = netdev_priv(dev);
2211        struct fw_iq_cmd cmd, rpl;
2212        int ret, iqandst, flsz = 0;
2213        int relaxed = !(adapter->flags & CXGB4VF_ROOT_NO_RELAXED_ORDERING);
2214
2215        /*
2216         * If we're using MSI interrupts and we're not initializing the
2217         * Forwarded Interrupt Queue itself, then set up this queue for
2218         * indirect interrupts to the Forwarded Interrupt Queue.  Obviously
2219         * the Forwarded Interrupt Queue must be set up before any other
2220         * ingress queue ...
2221         */
2222        if ((adapter->flags & CXGB4VF_USING_MSI) &&
2223            rspq != &adapter->sge.intrq) {
2224                iqandst = SGE_INTRDST_IQ;
2225                intr_dest = adapter->sge.intrq.abs_id;
2226        } else
2227                iqandst = SGE_INTRDST_PCI;
2228
2229        /*
2230         * Allocate the hardware ring for the Response Queue.  The size needs
2231         * to be a multiple of 16 which includes the mandatory status entry
2232         * (regardless of whether the Status Page capabilities are enabled or
2233         * not).
2234         */
2235        rspq->size = roundup(rspq->size, 16);
2236        rspq->desc = alloc_ring(adapter->pdev_dev, rspq->size, rspq->iqe_len,
2237                                0, &rspq->phys_addr, NULL, 0);
2238        if (!rspq->desc)
2239                return -ENOMEM;
2240
2241        /*
2242         * Fill in the Ingress Queue Command.  Note: Ideally this code would
2243         * be in t4vf_hw.c but there are so many parameters and dependencies
2244         * on our Linux SGE state that we would end up having to pass tons of
2245         * parameters.  We'll have to think about how this might be migrated
2246         * into OS-independent common code ...
2247         */
2248        memset(&cmd, 0, sizeof(cmd));
2249        cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_IQ_CMD) |
2250                                    FW_CMD_REQUEST_F |
2251                                    FW_CMD_WRITE_F |
2252                                    FW_CMD_EXEC_F);
2253        cmd.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_ALLOC_F |
2254                                         FW_IQ_CMD_IQSTART_F |
2255                                         FW_LEN16(cmd));
2256        cmd.type_to_iqandstindex =
2257                cpu_to_be32(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) |
2258                            FW_IQ_CMD_IQASYNCH_V(iqasynch) |
2259                            FW_IQ_CMD_VIID_V(pi->viid) |
2260                            FW_IQ_CMD_IQANDST_V(iqandst) |
2261                            FW_IQ_CMD_IQANUS_V(1) |
2262                            FW_IQ_CMD_IQANUD_V(SGE_UPDATEDEL_INTR) |
2263                            FW_IQ_CMD_IQANDSTINDEX_V(intr_dest));
2264        cmd.iqdroprss_to_iqesize =
2265                cpu_to_be16(FW_IQ_CMD_IQPCIECH_V(pi->port_id) |
2266                            FW_IQ_CMD_IQGTSMODE_F |
2267                            FW_IQ_CMD_IQINTCNTTHRESH_V(rspq->pktcnt_idx) |
2268                            FW_IQ_CMD_IQESIZE_V(ilog2(rspq->iqe_len) - 4));
2269        cmd.iqsize = cpu_to_be16(rspq->size);
2270        cmd.iqaddr = cpu_to_be64(rspq->phys_addr);
2271
2272        if (fl) {
2273                unsigned int chip_ver =
2274                        CHELSIO_CHIP_VERSION(adapter->params.chip);
2275                /*
2276                 * Allocate the ring for the hardware free list (with space
2277                 * for its status page) along with the associated software
2278                 * descriptor ring.  The free list size needs to be a multiple
2279                 * of the Egress Queue Unit and at least 2 Egress Units larger
2280                 * than the SGE's Egress Congrestion Threshold
2281                 * (fl_starve_thres - 1).
2282                 */
2283                if (fl->size < s->fl_starve_thres - 1 + 2 * FL_PER_EQ_UNIT)
2284                        fl->size = s->fl_starve_thres - 1 + 2 * FL_PER_EQ_UNIT;
2285                fl->size = roundup(fl->size, FL_PER_EQ_UNIT);
2286                fl->desc = alloc_ring(adapter->pdev_dev, fl->size,
2287                                      sizeof(__be64), sizeof(struct rx_sw_desc),
2288                                      &fl->addr, &fl->sdesc, s->stat_len);
2289                if (!fl->desc) {
2290                        ret = -ENOMEM;
2291                        goto err;
2292                }
2293
2294                /*
2295                 * Calculate the size of the hardware free list ring plus
2296                 * Status Page (which the SGE will place after the end of the
2297                 * free list ring) in Egress Queue Units.
2298                 */
2299                flsz = (fl->size / FL_PER_EQ_UNIT +
2300                        s->stat_len / EQ_UNIT);
2301
2302                /*
2303                 * Fill in all the relevant firmware Ingress Queue Command
2304                 * fields for the free list.
2305                 */
2306                cmd.iqns_to_fl0congen =
2307                        cpu_to_be32(
2308                                FW_IQ_CMD_FL0HOSTFCMODE_V(SGE_HOSTFCMODE_NONE) |
2309                                FW_IQ_CMD_FL0PACKEN_F |
2310                                FW_IQ_CMD_FL0FETCHRO_V(relaxed) |
2311                                FW_IQ_CMD_FL0DATARO_V(relaxed) |
2312                                FW_IQ_CMD_FL0PADEN_F);
2313
2314                /* In T6, for egress queue type FL there is internal overhead
2315                 * of 16B for header going into FLM module.  Hence the maximum
2316                 * allowed burst size is 448 bytes.  For T4/T5, the hardware
2317                 * doesn't coalesce fetch requests if more than 64 bytes of
2318                 * Free List pointers are provided, so we use a 128-byte Fetch
2319                 * Burst Minimum there (T6 implements coalescing so we can use
2320                 * the smaller 64-byte value there).
2321                 */
2322                cmd.fl0dcaen_to_fl0cidxfthresh =
2323                        cpu_to_be16(
2324                                FW_IQ_CMD_FL0FBMIN_V(chip_ver <= CHELSIO_T5
2325                                                     ? FETCHBURSTMIN_128B_X
2326                                                     : FETCHBURSTMIN_64B_T6_X) |
2327                                FW_IQ_CMD_FL0FBMAX_V((chip_ver <= CHELSIO_T5) ?
2328                                                     FETCHBURSTMAX_512B_X :
2329                                                     FETCHBURSTMAX_256B_X));
2330                cmd.fl0size = cpu_to_be16(flsz);
2331                cmd.fl0addr = cpu_to_be64(fl->addr);
2332        }
2333
2334        /*
2335         * Issue the firmware Ingress Queue Command and extract the results if
2336         * it completes successfully.
2337         */
2338        ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);
2339        if (ret)
2340                goto err;
2341
2342        netif_napi_add(dev, &rspq->napi, napi_rx_handler, 64);
2343        rspq->cur_desc = rspq->desc;
2344        rspq->cidx = 0;
2345        rspq->gen = 1;
2346        rspq->next_intr_params = rspq->intr_params;
2347        rspq->cntxt_id = be16_to_cpu(rpl.iqid);
2348        rspq->bar2_addr = bar2_address(adapter,
2349                                       rspq->cntxt_id,
2350                                       T4_BAR2_QTYPE_INGRESS,
2351                                       &rspq->bar2_qid);
2352        rspq->abs_id = be16_to_cpu(rpl.physiqid);
2353        rspq->size--;                   /* subtract status entry */
2354        rspq->adapter = adapter;
2355        rspq->netdev = dev;
2356        rspq->handler = hnd;
2357
2358        /* set offset to -1 to distinguish ingress queues without FL */
2359        rspq->offset = fl ? 0 : -1;
2360
2361        if (fl) {
2362                fl->cntxt_id = be16_to_cpu(rpl.fl0id);
2363                fl->avail = 0;
2364                fl->pend_cred = 0;
2365                fl->pidx = 0;
2366                fl->cidx = 0;
2367                fl->alloc_failed = 0;
2368                fl->large_alloc_failed = 0;
2369                fl->starving = 0;
2370
2371                /* Note, we must initialize the BAR2 Free List User Doorbell
2372                 * information before refilling the Free List!
2373                 */
2374                fl->bar2_addr = bar2_address(adapter,
2375                                             fl->cntxt_id,
2376                                             T4_BAR2_QTYPE_EGRESS,
2377                                             &fl->bar2_qid);
2378
2379                refill_fl(adapter, fl, fl_cap(fl), GFP_KERNEL);
2380        }
2381
2382        return 0;
2383
2384err:
2385        /*
2386         * An error occurred.  Clean up our partial allocation state and
2387         * return the error.
2388         */
2389        if (rspq->desc) {
2390                dma_free_coherent(adapter->pdev_dev, rspq->size * rspq->iqe_len,
2391                                  rspq->desc, rspq->phys_addr);
2392                rspq->desc = NULL;
2393        }
2394        if (fl && fl->desc) {
2395                kfree(fl->sdesc);
2396                fl->sdesc = NULL;
2397                dma_free_coherent(adapter->pdev_dev, flsz * EQ_UNIT,
2398                                  fl->desc, fl->addr);
2399                fl->desc = NULL;
2400        }
2401        return ret;
2402}
2403
2404/**
2405 *      t4vf_sge_alloc_eth_txq - allocate an SGE Ethernet TX Queue
2406 *      @adapter: the adapter
2407 *      @txq: pointer to the new txq to be filled in
2408 *      @dev: the network device
2409 *      @devq: the network TX queue associated with the new txq
2410 *      @iqid: the relative ingress queue ID to which events relating to
2411 *              the new txq should be directed
2412 */
2413int t4vf_sge_alloc_eth_txq(struct adapter *adapter, struct sge_eth_txq *txq,
2414                           struct net_device *dev, struct netdev_queue *devq,
2415                           unsigned int iqid)
2416{
2417        unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
2418        struct port_info *pi = netdev_priv(dev);
2419        struct fw_eq_eth_cmd cmd, rpl;
2420        struct sge *s = &adapter->sge;
2421        int ret, nentries;
2422
2423        /*
2424         * Calculate the size of the hardware TX Queue (including the Status
2425         * Page on the end of the TX Queue) in units of TX Descriptors.
2426         */
2427        nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
2428
2429        /*
2430         * Allocate the hardware ring for the TX ring (with space for its
2431         * status page) along with the associated software descriptor ring.
2432         */
2433        txq->q.desc = alloc_ring(adapter->pdev_dev, txq->q.size,
2434                                 sizeof(struct tx_desc),
2435                                 sizeof(struct tx_sw_desc),
2436                                 &txq->q.phys_addr, &txq->q.sdesc, s->stat_len);
2437        if (!txq->q.desc)
2438                return -ENOMEM;
2439
2440        /*
2441         * Fill in the Egress Queue Command.  Note: As with the direct use of
2442         * the firmware Ingress Queue COmmand above in our RXQ allocation
2443         * routine, ideally, this code would be in t4vf_hw.c.  Again, we'll
2444         * have to see if there's some reasonable way to parameterize it
2445         * into the common code ...
2446         */
2447        memset(&cmd, 0, sizeof(cmd));
2448        cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_EQ_ETH_CMD) |
2449                                    FW_CMD_REQUEST_F |
2450                                    FW_CMD_WRITE_F |
2451                                    FW_CMD_EXEC_F);
2452        cmd.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_ALLOC_F |
2453                                         FW_EQ_ETH_CMD_EQSTART_F |
2454                                         FW_LEN16(cmd));
2455        cmd.autoequiqe_to_viid = cpu_to_be32(FW_EQ_ETH_CMD_AUTOEQUEQE_F |
2456                                             FW_EQ_ETH_CMD_VIID_V(pi->viid));
2457        cmd.fetchszm_to_iqid =
2458                cpu_to_be32(FW_EQ_ETH_CMD_HOSTFCMODE_V(SGE_HOSTFCMODE_STPG) |
2459                            FW_EQ_ETH_CMD_PCIECHN_V(pi->port_id) |
2460                            FW_EQ_ETH_CMD_IQID_V(iqid));
2461        cmd.dcaen_to_eqsize =
2462                cpu_to_be32(FW_EQ_ETH_CMD_FBMIN_V(chip_ver <= CHELSIO_T5
2463                                                  ? FETCHBURSTMIN_64B_X
2464                                                  : FETCHBURSTMIN_64B_T6_X) |
2465                            FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
2466                            FW_EQ_ETH_CMD_CIDXFTHRESH_V(
2467                                                CIDXFLUSHTHRESH_32_X) |
2468                            FW_EQ_ETH_CMD_EQSIZE_V(nentries));
2469        cmd.eqaddr = cpu_to_be64(txq->q.phys_addr);
2470
2471        /*
2472         * Issue the firmware Egress Queue Command and extract the results if
2473         * it completes successfully.
2474         */
2475        ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl);
2476        if (ret) {
2477                /*
2478                 * The girmware Ingress Queue Command failed for some reason.
2479                 * Free up our partial allocation state and return the error.
2480                 */
2481                kfree(txq->q.sdesc);
2482                txq->q.sdesc = NULL;
2483                dma_free_coherent(adapter->pdev_dev,
2484                                  nentries * sizeof(struct tx_desc),
2485                                  txq->q.desc, txq->q.phys_addr);
2486                txq->q.desc = NULL;
2487                return ret;
2488        }
2489
2490        txq->q.in_use = 0;
2491        txq->q.cidx = 0;
2492        txq->q.pidx = 0;
2493        txq->q.stat = (void *)&txq->q.desc[txq->q.size];
2494        txq->q.cntxt_id = FW_EQ_ETH_CMD_EQID_G(be32_to_cpu(rpl.eqid_pkd));
2495        txq->q.bar2_addr = bar2_address(adapter,
2496                                        txq->q.cntxt_id,
2497                                        T4_BAR2_QTYPE_EGRESS,
2498                                        &txq->q.bar2_qid);
2499        txq->q.abs_id =
2500                FW_EQ_ETH_CMD_PHYSEQID_G(be32_to_cpu(rpl.physeqid_pkd));
2501        txq->txq = devq;
2502        txq->tso = 0;
2503        txq->tx_cso = 0;
2504        txq->vlan_ins = 0;
2505        txq->q.stops = 0;
2506        txq->q.restarts = 0;
2507        txq->mapping_err = 0;
2508        return 0;
2509}
2510
2511/*
2512 * Free the DMA map resources associated with a TX queue.
2513 */
2514static void free_txq(struct adapter *adapter, struct sge_txq *tq)
2515{
2516        struct sge *s = &adapter->sge;
2517
2518        dma_free_coherent(adapter->pdev_dev,
2519                          tq->size * sizeof(*tq->desc) + s->stat_len,
2520                          tq->desc, tq->phys_addr);
2521        tq->cntxt_id = 0;
2522        tq->sdesc = NULL;
2523        tq->desc = NULL;
2524}
2525
2526/*
2527 * Free the resources associated with a response queue (possibly including a
2528 * free list).
2529 */
2530static void free_rspq_fl(struct adapter *adapter, struct sge_rspq *rspq,
2531                         struct sge_fl *fl)
2532{
2533        struct sge *s = &adapter->sge;
2534        unsigned int flid = fl ? fl->cntxt_id : 0xffff;
2535
2536        t4vf_iq_free(adapter, FW_IQ_TYPE_FL_INT_CAP,
2537                     rspq->cntxt_id, flid, 0xffff);
2538        dma_free_coherent(adapter->pdev_dev, (rspq->size + 1) * rspq->iqe_len,
2539                          rspq->desc, rspq->phys_addr);
2540        netif_napi_del(&rspq->napi);
2541        rspq->netdev = NULL;
2542        rspq->cntxt_id = 0;
2543        rspq->abs_id = 0;
2544        rspq->desc = NULL;
2545
2546        if (fl) {
2547                free_rx_bufs(adapter, fl, fl->avail);
2548                dma_free_coherent(adapter->pdev_dev,
2549                                  fl->size * sizeof(*fl->desc) + s->stat_len,
2550                                  fl->desc, fl->addr);
2551                kfree(fl->sdesc);
2552                fl->sdesc = NULL;
2553                fl->cntxt_id = 0;
2554                fl->desc = NULL;
2555        }
2556}
2557
2558/**
2559 *      t4vf_free_sge_resources - free SGE resources
2560 *      @adapter: the adapter
2561 *
2562 *      Frees resources used by the SGE queue sets.
2563 */
2564void t4vf_free_sge_resources(struct adapter *adapter)
2565{
2566        struct sge *s = &adapter->sge;
2567        struct sge_eth_rxq *rxq = s->ethrxq;
2568        struct sge_eth_txq *txq = s->ethtxq;
2569        struct sge_rspq *evtq = &s->fw_evtq;
2570        struct sge_rspq *intrq = &s->intrq;
2571        int qs;
2572
2573        for (qs = 0; qs < adapter->sge.ethqsets; qs++, rxq++, txq++) {
2574                if (rxq->rspq.desc)
2575                        free_rspq_fl(adapter, &rxq->rspq, &rxq->fl);
2576                if (txq->q.desc) {
2577                        t4vf_eth_eq_free(adapter, txq->q.cntxt_id);
2578                        free_tx_desc(adapter, &txq->q, txq->q.in_use, true);
2579                        kfree(txq->q.sdesc);
2580                        free_txq(adapter, &txq->q);
2581                }
2582        }
2583        if (evtq->desc)
2584                free_rspq_fl(adapter, evtq, NULL);
2585        if (intrq->desc)
2586                free_rspq_fl(adapter, intrq, NULL);
2587}
2588
2589/**
2590 *      t4vf_sge_start - enable SGE operation
2591 *      @adapter: the adapter
2592 *
2593 *      Start tasklets and timers associated with the DMA engine.
2594 */
2595void t4vf_sge_start(struct adapter *adapter)
2596{
2597        adapter->sge.ethtxq_rover = 0;
2598        mod_timer(&adapter->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);
2599        mod_timer(&adapter->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);
2600}
2601
2602/**
2603 *      t4vf_sge_stop - disable SGE operation
2604 *      @adapter: the adapter
2605 *
2606 *      Stop tasklets and timers associated with the DMA engine.  Note that
2607 *      this is effective only if measures have been taken to disable any HW
2608 *      events that may restart them.
2609 */
2610void t4vf_sge_stop(struct adapter *adapter)
2611{
2612        struct sge *s = &adapter->sge;
2613
2614        if (s->rx_timer.function)
2615                del_timer_sync(&s->rx_timer);
2616        if (s->tx_timer.function)
2617                del_timer_sync(&s->tx_timer);
2618}
2619
2620/**
2621 *      t4vf_sge_init - initialize SGE
2622 *      @adapter: the adapter
2623 *
2624 *      Performs SGE initialization needed every time after a chip reset.
2625 *      We do not initialize any of the queue sets here, instead the driver
2626 *      top-level must request those individually.  We also do not enable DMA
2627 *      here, that should be done after the queues have been set up.
2628 */
2629int t4vf_sge_init(struct adapter *adapter)
2630{
2631        struct sge_params *sge_params = &adapter->params.sge;
2632        u32 fl_small_pg = sge_params->sge_fl_buffer_size[0];
2633        u32 fl_large_pg = sge_params->sge_fl_buffer_size[1];
2634        struct sge *s = &adapter->sge;
2635
2636        /*
2637         * Start by vetting the basic SGE parameters which have been set up by
2638         * the Physical Function Driver.  Ideally we should be able to deal
2639         * with _any_ configuration.  Practice is different ...
2640         */
2641
2642        /* We only bother using the Large Page logic if the Large Page Buffer
2643         * is larger than our Page Size Buffer.
2644         */
2645        if (fl_large_pg <= fl_small_pg)
2646                fl_large_pg = 0;
2647
2648        /* The Page Size Buffer must be exactly equal to our Page Size and the
2649         * Large Page Size Buffer should be 0 (per above) or a power of 2.
2650         */
2651        if (fl_small_pg != PAGE_SIZE ||
2652            (fl_large_pg & (fl_large_pg - 1)) != 0) {
2653                dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n",
2654                        fl_small_pg, fl_large_pg);
2655                return -EINVAL;
2656        }
2657        if ((sge_params->sge_control & RXPKTCPLMODE_F) !=
2658            RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) {
2659                dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n");
2660                return -EINVAL;
2661        }
2662
2663        /*
2664         * Now translate the adapter parameters into our internal forms.
2665         */
2666        if (fl_large_pg)
2667                s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
2668        s->stat_len = ((sge_params->sge_control & EGRSTATUSPAGESIZE_F)
2669                        ? 128 : 64);
2670        s->pktshift = PKTSHIFT_G(sge_params->sge_control);
2671        s->fl_align = t4vf_fl_pkt_align(adapter);
2672
2673        /* A FL with <= fl_starve_thres buffers is starving and a periodic
2674         * timer will attempt to refill it.  This needs to be larger than the
2675         * SGE's Egress Congestion Threshold.  If it isn't, then we can get
2676         * stuck waiting for new packets while the SGE is waiting for us to
2677         * give it more Free List entries.  (Note that the SGE's Egress
2678         * Congestion Threshold is in units of 2 Free List pointers.)
2679         */
2680        switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
2681        case CHELSIO_T4:
2682                s->fl_starve_thres =
2683                   EGRTHRESHOLD_G(sge_params->sge_congestion_control);
2684                break;
2685        case CHELSIO_T5:
2686                s->fl_starve_thres =
2687                   EGRTHRESHOLDPACKING_G(sge_params->sge_congestion_control);
2688                break;
2689        case CHELSIO_T6:
2690        default:
2691                s->fl_starve_thres =
2692                   T6_EGRTHRESHOLDPACKING_G(sge_params->sge_congestion_control);
2693                break;
2694        }
2695        s->fl_starve_thres = s->fl_starve_thres * 2 + 1;
2696
2697        /*
2698         * Set up tasklet timers.
2699         */
2700        timer_setup(&s->rx_timer, sge_rx_timer_cb, 0);
2701        timer_setup(&s->tx_timer, sge_tx_timer_cb, 0);
2702
2703        /*
2704         * Initialize Forwarded Interrupt Queue lock.
2705         */
2706        spin_lock_init(&s->intrq_lock);
2707
2708        return 0;
2709}
2710