linux/drivers/net/ethernet/chelsio/cxgb4/sge.c
<<
>>
Prefs
   1/*
   2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3 *
   4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 */
  34
  35#include <linux/skbuff.h>
  36#include <linux/netdevice.h>
  37#include <linux/etherdevice.h>
  38#include <linux/if_vlan.h>
  39#include <linux/ip.h>
  40#include <linux/dma-mapping.h>
  41#include <linux/jiffies.h>
  42#include <linux/prefetch.h>
  43#include <linux/export.h>
  44#include <net/xfrm.h>
  45#include <net/ipv6.h>
  46#include <net/tcp.h>
  47#include <net/busy_poll.h>
  48#include "cxgb4.h"
  49#include "t4_regs.h"
  50#include "t4_values.h"
  51#include "t4_msg.h"
  52#include "t4fw_api.h"
  53#include "cxgb4_ptp.h"
  54#include "cxgb4_uld.h"
  55
  56/*
  57 * Rx buffer size.  We use largish buffers if possible but settle for single
  58 * pages under memory shortage.
  59 */
  60#if PAGE_SHIFT >= 16
  61# define FL_PG_ORDER 0
  62#else
  63# define FL_PG_ORDER (16 - PAGE_SHIFT)
  64#endif
  65
  66/* RX_PULL_LEN should be <= RX_COPY_THRES */
  67#define RX_COPY_THRES    256
  68#define RX_PULL_LEN      128
  69
  70/*
  71 * Main body length for sk_buffs used for Rx Ethernet packets with fragments.
  72 * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room.
  73 */
  74#define RX_PKT_SKB_LEN   512
  75
  76/*
  77 * Max number of Tx descriptors we clean up at a time.  Should be modest as
  78 * freeing skbs isn't cheap and it happens while holding locks.  We just need
  79 * to free packets faster than they arrive, we eventually catch up and keep
  80 * the amortized cost reasonable.  Must be >= 2 * TXQ_STOP_THRES.
  81 */
  82#define MAX_TX_RECLAIM 16
  83
  84/*
  85 * Max number of Rx buffers we replenish at a time.  Again keep this modest,
  86 * allocating buffers isn't cheap either.
  87 */
  88#define MAX_RX_REFILL 16U
  89
  90/*
  91 * Period of the Rx queue check timer.  This timer is infrequent as it has
  92 * something to do only when the system experiences severe memory shortage.
  93 */
  94#define RX_QCHECK_PERIOD (HZ / 2)
  95
  96/*
  97 * Period of the Tx queue check timer.
  98 */
  99#define TX_QCHECK_PERIOD (HZ / 2)
 100
 101/*
 102 * Max number of Tx descriptors to be reclaimed by the Tx timer.
 103 */
 104#define MAX_TIMER_TX_RECLAIM 100
 105
 106/*
 107 * Timer index used when backing off due to memory shortage.
 108 */
 109#define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
 110
 111/*
 112 * Suspension threshold for non-Ethernet Tx queues.  We require enough room
 113 * for a full sized WR.
 114 */
 115#define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc))
 116
 117/*
 118 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
 119 * into a WR.
 120 */
 121#define MAX_IMM_TX_PKT_LEN 256
 122
 123/*
 124 * Max size of a WR sent through a control Tx queue.
 125 */
 126#define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
 127
 128struct rx_sw_desc {                /* SW state per Rx descriptor */
 129        struct page *page;
 130        dma_addr_t dma_addr;
 131};
 132
 133/*
 134 * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb
 135 * buffer).  We currently only support two sizes for 1500- and 9000-byte MTUs.
 136 * We could easily support more but there doesn't seem to be much need for
 137 * that ...
 138 */
 139#define FL_MTU_SMALL 1500
 140#define FL_MTU_LARGE 9000
 141
 142static inline unsigned int fl_mtu_bufsize(struct adapter *adapter,
 143                                          unsigned int mtu)
 144{
 145        struct sge *s = &adapter->sge;
 146
 147        return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align);
 148}
 149
 150#define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
 151#define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)
 152
 153/*
 154 * Bits 0..3 of rx_sw_desc.dma_addr have special meaning.  The hardware uses
 155 * these to specify the buffer size as an index into the SGE Free List Buffer
 156 * Size register array.  We also use bit 4, when the buffer has been unmapped
 157 * for DMA, but this is of course never sent to the hardware and is only used
 158 * to prevent double unmappings.  All of the above requires that the Free List
 159 * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are
 160 * 32-byte or or a power of 2 greater in alignment.  Since the SGE's minimal
 161 * Free List Buffer alignment is 32 bytes, this works out for us ...
 162 */
 163enum {
 164        RX_BUF_FLAGS     = 0x1f,   /* bottom five bits are special */
 165        RX_BUF_SIZE      = 0x0f,   /* bottom three bits are for buf sizes */
 166        RX_UNMAPPED_BUF  = 0x10,   /* buffer is not mapped */
 167
 168        /*
 169         * XXX We shouldn't depend on being able to use these indices.
 170         * XXX Especially when some other Master PF has initialized the
 171         * XXX adapter or we use the Firmware Configuration File.  We
 172         * XXX should really search through the Host Buffer Size register
 173         * XXX array for the appropriately sized buffer indices.
 174         */
 175        RX_SMALL_PG_BUF  = 0x0,   /* small (PAGE_SIZE) page buffer */
 176        RX_LARGE_PG_BUF  = 0x1,   /* buffer large (FL_PG_ORDER) page buffer */
 177
 178        RX_SMALL_MTU_BUF = 0x2,   /* small MTU buffer */
 179        RX_LARGE_MTU_BUF = 0x3,   /* large MTU buffer */
 180};
 181
 182static int timer_pkt_quota[] = {1, 1, 2, 3, 4, 5};
 183#define MIN_NAPI_WORK  1
 184
 185static inline dma_addr_t get_buf_addr(const struct rx_sw_desc *d)
 186{
 187        return d->dma_addr & ~(dma_addr_t)RX_BUF_FLAGS;
 188}
 189
 190static inline bool is_buf_mapped(const struct rx_sw_desc *d)
 191{
 192        return !(d->dma_addr & RX_UNMAPPED_BUF);
 193}
 194
 195/**
 196 *      txq_avail - return the number of available slots in a Tx queue
 197 *      @q: the Tx queue
 198 *
 199 *      Returns the number of descriptors in a Tx queue available to write new
 200 *      packets.
 201 */
 202static inline unsigned int txq_avail(const struct sge_txq *q)
 203{
 204        return q->size - 1 - q->in_use;
 205}
 206
 207/**
 208 *      fl_cap - return the capacity of a free-buffer list
 209 *      @fl: the FL
 210 *
 211 *      Returns the capacity of a free-buffer list.  The capacity is less than
 212 *      the size because one descriptor needs to be left unpopulated, otherwise
 213 *      HW will think the FL is empty.
 214 */
 215static inline unsigned int fl_cap(const struct sge_fl *fl)
 216{
 217        return fl->size - 8;   /* 1 descriptor = 8 buffers */
 218}
 219
 220/**
 221 *      fl_starving - return whether a Free List is starving.
 222 *      @adapter: pointer to the adapter
 223 *      @fl: the Free List
 224 *
 225 *      Tests specified Free List to see whether the number of buffers
 226 *      available to the hardware has falled below our "starvation"
 227 *      threshold.
 228 */
 229static inline bool fl_starving(const struct adapter *adapter,
 230                               const struct sge_fl *fl)
 231{
 232        const struct sge *s = &adapter->sge;
 233
 234        return fl->avail - fl->pend_cred <= s->fl_starve_thres;
 235}
 236
 237int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
 238                  dma_addr_t *addr)
 239{
 240        const skb_frag_t *fp, *end;
 241        const struct skb_shared_info *si;
 242
 243        *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
 244        if (dma_mapping_error(dev, *addr))
 245                goto out_err;
 246
 247        si = skb_shinfo(skb);
 248        end = &si->frags[si->nr_frags];
 249
 250        for (fp = si->frags; fp < end; fp++) {
 251                *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp),
 252                                           DMA_TO_DEVICE);
 253                if (dma_mapping_error(dev, *addr))
 254                        goto unwind;
 255        }
 256        return 0;
 257
 258unwind:
 259        while (fp-- > si->frags)
 260                dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
 261
 262        dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE);
 263out_err:
 264        return -ENOMEM;
 265}
 266EXPORT_SYMBOL(cxgb4_map_skb);
 267
 268#ifdef CONFIG_NEED_DMA_MAP_STATE
 269static void unmap_skb(struct device *dev, const struct sk_buff *skb,
 270                      const dma_addr_t *addr)
 271{
 272        const skb_frag_t *fp, *end;
 273        const struct skb_shared_info *si;
 274
 275        dma_unmap_single(dev, *addr++, skb_headlen(skb), DMA_TO_DEVICE);
 276
 277        si = skb_shinfo(skb);
 278        end = &si->frags[si->nr_frags];
 279        for (fp = si->frags; fp < end; fp++)
 280                dma_unmap_page(dev, *addr++, skb_frag_size(fp), DMA_TO_DEVICE);
 281}
 282
 283/**
 284 *      deferred_unmap_destructor - unmap a packet when it is freed
 285 *      @skb: the packet
 286 *
 287 *      This is the packet destructor used for Tx packets that need to remain
 288 *      mapped until they are freed rather than until their Tx descriptors are
 289 *      freed.
 290 */
 291static void deferred_unmap_destructor(struct sk_buff *skb)
 292{
 293        unmap_skb(skb->dev->dev.parent, skb, (dma_addr_t *)skb->head);
 294}
 295#endif
 296
 297static void unmap_sgl(struct device *dev, const struct sk_buff *skb,
 298                      const struct ulptx_sgl *sgl, const struct sge_txq *q)
 299{
 300        const struct ulptx_sge_pair *p;
 301        unsigned int nfrags = skb_shinfo(skb)->nr_frags;
 302
 303        if (likely(skb_headlen(skb)))
 304                dma_unmap_single(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
 305                                 DMA_TO_DEVICE);
 306        else {
 307                dma_unmap_page(dev, be64_to_cpu(sgl->addr0), ntohl(sgl->len0),
 308                               DMA_TO_DEVICE);
 309                nfrags--;
 310        }
 311
 312        /*
 313         * the complexity below is because of the possibility of a wrap-around
 314         * in the middle of an SGL
 315         */
 316        for (p = sgl->sge; nfrags >= 2; nfrags -= 2) {
 317                if (likely((u8 *)(p + 1) <= (u8 *)q->stat)) {
 318unmap:                  dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
 319                                       ntohl(p->len[0]), DMA_TO_DEVICE);
 320                        dma_unmap_page(dev, be64_to_cpu(p->addr[1]),
 321                                       ntohl(p->len[1]), DMA_TO_DEVICE);
 322                        p++;
 323                } else if ((u8 *)p == (u8 *)q->stat) {
 324                        p = (const struct ulptx_sge_pair *)q->desc;
 325                        goto unmap;
 326                } else if ((u8 *)p + 8 == (u8 *)q->stat) {
 327                        const __be64 *addr = (const __be64 *)q->desc;
 328
 329                        dma_unmap_page(dev, be64_to_cpu(addr[0]),
 330                                       ntohl(p->len[0]), DMA_TO_DEVICE);
 331                        dma_unmap_page(dev, be64_to_cpu(addr[1]),
 332                                       ntohl(p->len[1]), DMA_TO_DEVICE);
 333                        p = (const struct ulptx_sge_pair *)&addr[2];
 334                } else {
 335                        const __be64 *addr = (const __be64 *)q->desc;
 336
 337                        dma_unmap_page(dev, be64_to_cpu(p->addr[0]),
 338                                       ntohl(p->len[0]), DMA_TO_DEVICE);
 339                        dma_unmap_page(dev, be64_to_cpu(addr[0]),
 340                                       ntohl(p->len[1]), DMA_TO_DEVICE);
 341                        p = (const struct ulptx_sge_pair *)&addr[1];
 342                }
 343        }
 344        if (nfrags) {
 345                __be64 addr;
 346
 347                if ((u8 *)p == (u8 *)q->stat)
 348                        p = (const struct ulptx_sge_pair *)q->desc;
 349                addr = (u8 *)p + 16 <= (u8 *)q->stat ? p->addr[0] :
 350                                                       *(const __be64 *)q->desc;
 351                dma_unmap_page(dev, be64_to_cpu(addr), ntohl(p->len[0]),
 352                               DMA_TO_DEVICE);
 353        }
 354}
 355
 356/**
 357 *      free_tx_desc - reclaims Tx descriptors and their buffers
 358 *      @adapter: the adapter
 359 *      @q: the Tx queue to reclaim descriptors from
 360 *      @n: the number of descriptors to reclaim
 361 *      @unmap: whether the buffers should be unmapped for DMA
 362 *
 363 *      Reclaims Tx descriptors from an SGE Tx queue and frees the associated
 364 *      Tx buffers.  Called with the Tx queue lock held.
 365 */
 366void free_tx_desc(struct adapter *adap, struct sge_txq *q,
 367                  unsigned int n, bool unmap)
 368{
 369        struct tx_sw_desc *d;
 370        unsigned int cidx = q->cidx;
 371        struct device *dev = adap->pdev_dev;
 372
 373        d = &q->sdesc[cidx];
 374        while (n--) {
 375                if (d->skb) {                       /* an SGL is present */
 376                        if (unmap)
 377                                unmap_sgl(dev, d->skb, d->sgl, q);
 378                        dev_consume_skb_any(d->skb);
 379                        d->skb = NULL;
 380                }
 381                ++d;
 382                if (++cidx == q->size) {
 383                        cidx = 0;
 384                        d = q->sdesc;
 385                }
 386        }
 387        q->cidx = cidx;
 388}
 389
 390/*
 391 * Return the number of reclaimable descriptors in a Tx queue.
 392 */
 393static inline int reclaimable(const struct sge_txq *q)
 394{
 395        int hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
 396        hw_cidx -= q->cidx;
 397        return hw_cidx < 0 ? hw_cidx + q->size : hw_cidx;
 398}
 399
 400/**
 401 *      cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors
 402 *      @adap: the adapter
 403 *      @q: the Tx queue to reclaim completed descriptors from
 404 *      @unmap: whether the buffers should be unmapped for DMA
 405 *
 406 *      Reclaims Tx descriptors that the SGE has indicated it has processed,
 407 *      and frees the associated buffers if possible.  Called with the Tx
 408 *      queue locked.
 409 */
 410inline void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
 411                                        bool unmap)
 412{
 413        int avail = reclaimable(q);
 414
 415        if (avail) {
 416                /*
 417                 * Limit the amount of clean up work we do at a time to keep
 418                 * the Tx lock hold time O(1).
 419                 */
 420                if (avail > MAX_TX_RECLAIM)
 421                        avail = MAX_TX_RECLAIM;
 422
 423                free_tx_desc(adap, q, avail, unmap);
 424                q->in_use -= avail;
 425        }
 426}
 427EXPORT_SYMBOL(cxgb4_reclaim_completed_tx);
 428
 429static inline int get_buf_size(struct adapter *adapter,
 430                               const struct rx_sw_desc *d)
 431{
 432        struct sge *s = &adapter->sge;
 433        unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE;
 434        int buf_size;
 435
 436        switch (rx_buf_size_idx) {
 437        case RX_SMALL_PG_BUF:
 438                buf_size = PAGE_SIZE;
 439                break;
 440
 441        case RX_LARGE_PG_BUF:
 442                buf_size = PAGE_SIZE << s->fl_pg_order;
 443                break;
 444
 445        case RX_SMALL_MTU_BUF:
 446                buf_size = FL_MTU_SMALL_BUFSIZE(adapter);
 447                break;
 448
 449        case RX_LARGE_MTU_BUF:
 450                buf_size = FL_MTU_LARGE_BUFSIZE(adapter);
 451                break;
 452
 453        default:
 454                BUG_ON(1);
 455        }
 456
 457        return buf_size;
 458}
 459
 460/**
 461 *      free_rx_bufs - free the Rx buffers on an SGE free list
 462 *      @adap: the adapter
 463 *      @q: the SGE free list to free buffers from
 464 *      @n: how many buffers to free
 465 *
 466 *      Release the next @n buffers on an SGE free-buffer Rx queue.   The
 467 *      buffers must be made inaccessible to HW before calling this function.
 468 */
 469static void free_rx_bufs(struct adapter *adap, struct sge_fl *q, int n)
 470{
 471        while (n--) {
 472                struct rx_sw_desc *d = &q->sdesc[q->cidx];
 473
 474                if (is_buf_mapped(d))
 475                        dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
 476                                       get_buf_size(adap, d),
 477                                       PCI_DMA_FROMDEVICE);
 478                put_page(d->page);
 479                d->page = NULL;
 480                if (++q->cidx == q->size)
 481                        q->cidx = 0;
 482                q->avail--;
 483        }
 484}
 485
 486/**
 487 *      unmap_rx_buf - unmap the current Rx buffer on an SGE free list
 488 *      @adap: the adapter
 489 *      @q: the SGE free list
 490 *
 491 *      Unmap the current buffer on an SGE free-buffer Rx queue.   The
 492 *      buffer must be made inaccessible to HW before calling this function.
 493 *
 494 *      This is similar to @free_rx_bufs above but does not free the buffer.
 495 *      Do note that the FL still loses any further access to the buffer.
 496 */
 497static void unmap_rx_buf(struct adapter *adap, struct sge_fl *q)
 498{
 499        struct rx_sw_desc *d = &q->sdesc[q->cidx];
 500
 501        if (is_buf_mapped(d))
 502                dma_unmap_page(adap->pdev_dev, get_buf_addr(d),
 503                               get_buf_size(adap, d), PCI_DMA_FROMDEVICE);
 504        d->page = NULL;
 505        if (++q->cidx == q->size)
 506                q->cidx = 0;
 507        q->avail--;
 508}
 509
 510static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
 511{
 512        if (q->pend_cred >= 8) {
 513                u32 val = adap->params.arch.sge_fl_db;
 514
 515                if (is_t4(adap->params.chip))
 516                        val |= PIDX_V(q->pend_cred / 8);
 517                else
 518                        val |= PIDX_T5_V(q->pend_cred / 8);
 519
 520                /* Make sure all memory writes to the Free List queue are
 521                 * committed before we tell the hardware about them.
 522                 */
 523                wmb();
 524
 525                /* If we don't have access to the new User Doorbell (T5+), use
 526                 * the old doorbell mechanism; otherwise use the new BAR2
 527                 * mechanism.
 528                 */
 529                if (unlikely(q->bar2_addr == NULL)) {
 530                        t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
 531                                     val | QID_V(q->cntxt_id));
 532                } else {
 533                        writel(val | QID_V(q->bar2_qid),
 534                               q->bar2_addr + SGE_UDB_KDOORBELL);
 535
 536                        /* This Write memory Barrier will force the write to
 537                         * the User Doorbell area to be flushed.
 538                         */
 539                        wmb();
 540                }
 541                q->pend_cred &= 7;
 542        }
 543}
 544
 545static inline void set_rx_sw_desc(struct rx_sw_desc *sd, struct page *pg,
 546                                  dma_addr_t mapping)
 547{
 548        sd->page = pg;
 549        sd->dma_addr = mapping;      /* includes size low bits */
 550}
 551
 552/**
 553 *      refill_fl - refill an SGE Rx buffer ring
 554 *      @adap: the adapter
 555 *      @q: the ring to refill
 556 *      @n: the number of new buffers to allocate
 557 *      @gfp: the gfp flags for the allocations
 558 *
 559 *      (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
 560 *      allocated with the supplied gfp flags.  The caller must assure that
 561 *      @n does not exceed the queue's capacity.  If afterwards the queue is
 562 *      found critically low mark it as starving in the bitmap of starving FLs.
 563 *
 564 *      Returns the number of buffers allocated.
 565 */
 566static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n,
 567                              gfp_t gfp)
 568{
 569        struct sge *s = &adap->sge;
 570        struct page *pg;
 571        dma_addr_t mapping;
 572        unsigned int cred = q->avail;
 573        __be64 *d = &q->desc[q->pidx];
 574        struct rx_sw_desc *sd = &q->sdesc[q->pidx];
 575        int node;
 576
 577#ifdef CONFIG_DEBUG_FS
 578        if (test_bit(q->cntxt_id - adap->sge.egr_start, adap->sge.blocked_fl))
 579                goto out;
 580#endif
 581
 582        gfp |= __GFP_NOWARN | __GFP_COLD;
 583        node = dev_to_node(adap->pdev_dev);
 584
 585        if (s->fl_pg_order == 0)
 586                goto alloc_small_pages;
 587
 588        /*
 589         * Prefer large buffers
 590         */
 591        while (n) {
 592                pg = alloc_pages_node(node, gfp | __GFP_COMP, s->fl_pg_order);
 593                if (unlikely(!pg)) {
 594                        q->large_alloc_failed++;
 595                        break;       /* fall back to single pages */
 596                }
 597
 598                mapping = dma_map_page(adap->pdev_dev, pg, 0,
 599                                       PAGE_SIZE << s->fl_pg_order,
 600                                       PCI_DMA_FROMDEVICE);
 601                if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
 602                        __free_pages(pg, s->fl_pg_order);
 603                        q->mapping_err++;
 604                        goto out;   /* do not try small pages for this error */
 605                }
 606                mapping |= RX_LARGE_PG_BUF;
 607                *d++ = cpu_to_be64(mapping);
 608
 609                set_rx_sw_desc(sd, pg, mapping);
 610                sd++;
 611
 612                q->avail++;
 613                if (++q->pidx == q->size) {
 614                        q->pidx = 0;
 615                        sd = q->sdesc;
 616                        d = q->desc;
 617                }
 618                n--;
 619        }
 620
 621alloc_small_pages:
 622        while (n--) {
 623                pg = alloc_pages_node(node, gfp, 0);
 624                if (unlikely(!pg)) {
 625                        q->alloc_failed++;
 626                        break;
 627                }
 628
 629                mapping = dma_map_page(adap->pdev_dev, pg, 0, PAGE_SIZE,
 630                                       PCI_DMA_FROMDEVICE);
 631                if (unlikely(dma_mapping_error(adap->pdev_dev, mapping))) {
 632                        put_page(pg);
 633                        q->mapping_err++;
 634                        goto out;
 635                }
 636                *d++ = cpu_to_be64(mapping);
 637
 638                set_rx_sw_desc(sd, pg, mapping);
 639                sd++;
 640
 641                q->avail++;
 642                if (++q->pidx == q->size) {
 643                        q->pidx = 0;
 644                        sd = q->sdesc;
 645                        d = q->desc;
 646                }
 647        }
 648
 649out:    cred = q->avail - cred;
 650        q->pend_cred += cred;
 651        ring_fl_db(adap, q);
 652
 653        if (unlikely(fl_starving(adap, q))) {
 654                smp_wmb();
 655                q->low++;
 656                set_bit(q->cntxt_id - adap->sge.egr_start,
 657                        adap->sge.starving_fl);
 658        }
 659
 660        return cred;
 661}
 662
 663static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
 664{
 665        refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail),
 666                  GFP_ATOMIC);
 667}
 668
 669/**
 670 *      alloc_ring - allocate resources for an SGE descriptor ring
 671 *      @dev: the PCI device's core device
 672 *      @nelem: the number of descriptors
 673 *      @elem_size: the size of each descriptor
 674 *      @sw_size: the size of the SW state associated with each ring element
 675 *      @phys: the physical address of the allocated ring
 676 *      @metadata: address of the array holding the SW state for the ring
 677 *      @stat_size: extra space in HW ring for status information
 678 *      @node: preferred node for memory allocations
 679 *
 680 *      Allocates resources for an SGE descriptor ring, such as Tx queues,
 681 *      free buffer lists, or response queues.  Each SGE ring requires
 682 *      space for its HW descriptors plus, optionally, space for the SW state
 683 *      associated with each HW entry (the metadata).  The function returns
 684 *      three values: the virtual address for the HW ring (the return value
 685 *      of the function), the bus address of the HW ring, and the address
 686 *      of the SW ring.
 687 */
 688static void *alloc_ring(struct device *dev, size_t nelem, size_t elem_size,
 689                        size_t sw_size, dma_addr_t *phys, void *metadata,
 690                        size_t stat_size, int node)
 691{
 692        size_t len = nelem * elem_size + stat_size;
 693        void *s = NULL;
 694        void *p = dma_zalloc_coherent(dev, len, phys, GFP_KERNEL);
 695
 696        if (!p)
 697                return NULL;
 698        if (sw_size) {
 699                s = kcalloc_node(sw_size, nelem, GFP_KERNEL, node);
 700
 701                if (!s) {
 702                        dma_free_coherent(dev, len, p, *phys);
 703                        return NULL;
 704                }
 705        }
 706        if (metadata)
 707                *(void **)metadata = s;
 708        return p;
 709}
 710
 711/**
 712 *      sgl_len - calculates the size of an SGL of the given capacity
 713 *      @n: the number of SGL entries
 714 *
 715 *      Calculates the number of flits needed for a scatter/gather list that
 716 *      can hold the given number of entries.
 717 */
 718static inline unsigned int sgl_len(unsigned int n)
 719{
 720        /* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
 721         * addresses.  The DSGL Work Request starts off with a 32-bit DSGL
 722         * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
 723         * repeated sequences of { Length[i], Length[i+1], Address[i],
 724         * Address[i+1] } (this ensures that all addresses are on 64-bit
 725         * boundaries).  If N is even, then Length[N+1] should be set to 0 and
 726         * Address[N+1] is omitted.
 727         *
 728         * The following calculation incorporates all of the above.  It's
 729         * somewhat hard to follow but, briefly: the "+2" accounts for the
 730         * first two flits which include the DSGL header, Length0 and
 731         * Address0; the "(3*(n-1))/2" covers the main body of list entries (3
 732         * flits for every pair of the remaining N) +1 if (n-1) is odd; and
 733         * finally the "+((n-1)&1)" adds the one remaining flit needed if
 734         * (n-1) is odd ...
 735         */
 736        n--;
 737        return (3 * n) / 2 + (n & 1) + 2;
 738}
 739
 740/**
 741 *      flits_to_desc - returns the num of Tx descriptors for the given flits
 742 *      @n: the number of flits
 743 *
 744 *      Returns the number of Tx descriptors needed for the supplied number
 745 *      of flits.
 746 */
 747static inline unsigned int flits_to_desc(unsigned int n)
 748{
 749        BUG_ON(n > SGE_MAX_WR_LEN / 8);
 750        return DIV_ROUND_UP(n, 8);
 751}
 752
 753/**
 754 *      is_eth_imm - can an Ethernet packet be sent as immediate data?
 755 *      @skb: the packet
 756 *
 757 *      Returns whether an Ethernet packet is small enough to fit as
 758 *      immediate data. Return value corresponds to headroom required.
 759 */
 760static inline int is_eth_imm(const struct sk_buff *skb, unsigned int chip_ver)
 761{
 762        int hdrlen = 0;
 763
 764        if (skb->encapsulation && skb_shinfo(skb)->gso_size &&
 765            chip_ver > CHELSIO_T5) {
 766                hdrlen = sizeof(struct cpl_tx_tnl_lso);
 767                hdrlen += sizeof(struct cpl_tx_pkt_core);
 768        } else {
 769                hdrlen = skb_shinfo(skb)->gso_size ?
 770                         sizeof(struct cpl_tx_pkt_lso_core) : 0;
 771                hdrlen += sizeof(struct cpl_tx_pkt);
 772        }
 773        if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
 774                return hdrlen;
 775        return 0;
 776}
 777
 778/**
 779 *      calc_tx_flits - calculate the number of flits for a packet Tx WR
 780 *      @skb: the packet
 781 *
 782 *      Returns the number of flits needed for a Tx WR for the given Ethernet
 783 *      packet, including the needed WR and CPL headers.
 784 */
 785static inline unsigned int calc_tx_flits(const struct sk_buff *skb,
 786                                         unsigned int chip_ver)
 787{
 788        unsigned int flits;
 789        int hdrlen = is_eth_imm(skb, chip_ver);
 790
 791        /* If the skb is small enough, we can pump it out as a work request
 792         * with only immediate data.  In that case we just have to have the
 793         * TX Packet header plus the skb data in the Work Request.
 794         */
 795
 796        if (hdrlen)
 797                return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
 798
 799        /* Otherwise, we're going to have to construct a Scatter gather list
 800         * of the skb body and fragments.  We also include the flits necessary
 801         * for the TX Packet Work Request and CPL.  We always have a firmware
 802         * Write Header (incorporated as part of the cpl_tx_pkt_lso and
 803         * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
 804         * message or, if we're doing a Large Send Offload, an LSO CPL message
 805         * with an embedded TX Packet Write CPL message.
 806         */
 807        flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
 808        if (skb_shinfo(skb)->gso_size) {
 809                if (skb->encapsulation && chip_ver > CHELSIO_T5)
 810                        hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
 811                                 sizeof(struct cpl_tx_tnl_lso);
 812                else
 813                        hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
 814                                 sizeof(struct cpl_tx_pkt_lso_core);
 815
 816                hdrlen += sizeof(struct cpl_tx_pkt_core);
 817                flits += (hdrlen / sizeof(__be64));
 818        } else {
 819                flits += (sizeof(struct fw_eth_tx_pkt_wr) +
 820                          sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
 821        }
 822        return flits;
 823}
 824
 825/**
 826 *      calc_tx_descs - calculate the number of Tx descriptors for a packet
 827 *      @skb: the packet
 828 *
 829 *      Returns the number of Tx descriptors needed for the given Ethernet
 830 *      packet, including the needed WR and CPL headers.
 831 */
 832static inline unsigned int calc_tx_descs(const struct sk_buff *skb,
 833                                         unsigned int chip_ver)
 834{
 835        return flits_to_desc(calc_tx_flits(skb, chip_ver));
 836}
 837
 838/**
 839 *      cxgb4_write_sgl - populate a scatter/gather list for a packet
 840 *      @skb: the packet
 841 *      @q: the Tx queue we are writing into
 842 *      @sgl: starting location for writing the SGL
 843 *      @end: points right after the end of the SGL
 844 *      @start: start offset into skb main-body data to include in the SGL
 845 *      @addr: the list of bus addresses for the SGL elements
 846 *
 847 *      Generates a gather list for the buffers that make up a packet.
 848 *      The caller must provide adequate space for the SGL that will be written.
 849 *      The SGL includes all of the packet's page fragments and the data in its
 850 *      main body except for the first @start bytes.  @sgl must be 16-byte
 851 *      aligned and within a Tx descriptor with available space.  @end points
 852 *      right after the end of the SGL but does not account for any potential
 853 *      wrap around, i.e., @end > @sgl.
 854 */
 855void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
 856                     struct ulptx_sgl *sgl, u64 *end, unsigned int start,
 857                     const dma_addr_t *addr)
 858{
 859        unsigned int i, len;
 860        struct ulptx_sge_pair *to;
 861        const struct skb_shared_info *si = skb_shinfo(skb);
 862        unsigned int nfrags = si->nr_frags;
 863        struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1];
 864
 865        len = skb_headlen(skb) - start;
 866        if (likely(len)) {
 867                sgl->len0 = htonl(len);
 868                sgl->addr0 = cpu_to_be64(addr[0] + start);
 869                nfrags++;
 870        } else {
 871                sgl->len0 = htonl(skb_frag_size(&si->frags[0]));
 872                sgl->addr0 = cpu_to_be64(addr[1]);
 873        }
 874
 875        sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
 876                              ULPTX_NSGE_V(nfrags));
 877        if (likely(--nfrags == 0))
 878                return;
 879        /*
 880         * Most of the complexity below deals with the possibility we hit the
 881         * end of the queue in the middle of writing the SGL.  For this case
 882         * only we create the SGL in a temporary buffer and then copy it.
 883         */
 884        to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge;
 885
 886        for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) {
 887                to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
 888                to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i]));
 889                to->addr[0] = cpu_to_be64(addr[i]);
 890                to->addr[1] = cpu_to_be64(addr[++i]);
 891        }
 892        if (nfrags) {
 893                to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i]));
 894                to->len[1] = cpu_to_be32(0);
 895                to->addr[0] = cpu_to_be64(addr[i + 1]);
 896        }
 897        if (unlikely((u8 *)end > (u8 *)q->stat)) {
 898                unsigned int part0 = (u8 *)q->stat - (u8 *)sgl->sge, part1;
 899
 900                if (likely(part0))
 901                        memcpy(sgl->sge, buf, part0);
 902                part1 = (u8 *)end - (u8 *)q->stat;
 903                memcpy(q->desc, (u8 *)buf + part0, part1);
 904                end = (void *)q->desc + part1;
 905        }
 906        if ((uintptr_t)end & 8)           /* 0-pad to multiple of 16 */
 907                *end = 0;
 908}
 909EXPORT_SYMBOL(cxgb4_write_sgl);
 910
 911/* This function copies 64 byte coalesced work request to
 912 * memory mapped BAR2 space(user space writes).
 913 * For coalesced WR SGE, fetches data from the FIFO instead of from Host.
 914 */
 915static void cxgb_pio_copy(u64 __iomem *dst, u64 *src)
 916{
 917        int count = 8;
 918
 919        while (count) {
 920                writeq(*src, dst);
 921                src++;
 922                dst++;
 923                count--;
 924        }
 925}
 926
 927/**
 928 *      cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell
 929 *      @adap: the adapter
 930 *      @q: the Tx queue
 931 *      @n: number of new descriptors to give to HW
 932 *
 933 *      Ring the doorbel for a Tx queue.
 934 */
 935inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
 936{
 937        /* Make sure that all writes to the TX Descriptors are committed
 938         * before we tell the hardware about them.
 939         */
 940        wmb();
 941
 942        /* If we don't have access to the new User Doorbell (T5+), use the old
 943         * doorbell mechanism; otherwise use the new BAR2 mechanism.
 944         */
 945        if (unlikely(q->bar2_addr == NULL)) {
 946                u32 val = PIDX_V(n);
 947                unsigned long flags;
 948
 949                /* For T4 we need to participate in the Doorbell Recovery
 950                 * mechanism.
 951                 */
 952                spin_lock_irqsave(&q->db_lock, flags);
 953                if (!q->db_disabled)
 954                        t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
 955                                     QID_V(q->cntxt_id) | val);
 956                else
 957                        q->db_pidx_inc += n;
 958                q->db_pidx = q->pidx;
 959                spin_unlock_irqrestore(&q->db_lock, flags);
 960        } else {
 961                u32 val = PIDX_T5_V(n);
 962
 963                /* T4 and later chips share the same PIDX field offset within
 964                 * the doorbell, but T5 and later shrank the field in order to
 965                 * gain a bit for Doorbell Priority.  The field was absurdly
 966                 * large in the first place (14 bits) so we just use the T5
 967                 * and later limits and warn if a Queue ID is too large.
 968                 */
 969                WARN_ON(val & DBPRIO_F);
 970
 971                /* If we're only writing a single TX Descriptor and we can use
 972                 * Inferred QID registers, we can use the Write Combining
 973                 * Gather Buffer; otherwise we use the simple doorbell.
 974                 */
 975                if (n == 1 && q->bar2_qid == 0) {
 976                        int index = (q->pidx
 977                                     ? (q->pidx - 1)
 978                                     : (q->size - 1));
 979                        u64 *wr = (u64 *)&q->desc[index];
 980                        
 981                        cxgb_pio_copy((u64 __iomem *)
 982                                      (q->bar2_addr + SGE_UDB_WCDOORBELL),
 983                                      wr);
 984                } else {
 985                        writel(val | QID_V(q->bar2_qid),
 986                               q->bar2_addr + SGE_UDB_KDOORBELL);               
 987}
 988
 989                /* This Write Memory Barrier will force the write to the User
 990                 * Doorbell area to be flushed.  This is needed to prevent
 991                 * writes on different CPUs for the same queue from hitting
 992                 * the adapter out of order.  This is required when some Work
 993                 * Requests take the Write Combine Gather Buffer path (user
 994                 * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
 995                 * take the traditional path where we simply increment the
 996                 * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
 997                 * hardware DMA read the actual Work Request.
 998                 */
 999                wmb();
1000        }
1001}
1002EXPORT_SYMBOL(cxgb4_ring_tx_db);
1003
1004/**
1005 *      cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors
1006 *      @skb: the packet
1007 *      @q: the Tx queue where the packet will be inlined
1008 *      @pos: starting position in the Tx queue where to inline the packet
1009 *
1010 *      Inline a packet's contents directly into Tx descriptors, starting at
1011 *      the given position within the Tx DMA ring.
1012 *      Most of the complexity of this operation is dealing with wrap arounds
1013 *      in the middle of the packet we want to inline.
1014 */
1015void cxgb4_inline_tx_skb(const struct sk_buff *skb,
1016                         const struct sge_txq *q, void *pos)
1017{
1018        int left = (void *)q->stat - pos;
1019        u64 *p;
1020
1021        if (likely(skb->len <= left)) {
1022                if (likely(!skb->data_len))
1023                        skb_copy_from_linear_data(skb, pos, skb->len);
1024                else
1025                        skb_copy_bits(skb, 0, pos, skb->len);
1026                pos += skb->len;
1027        } else {
1028                skb_copy_bits(skb, 0, pos, left);
1029                skb_copy_bits(skb, left, q->desc, skb->len - left);
1030                pos = (void *)q->desc + (skb->len - left);
1031        }
1032
1033        /* 0-pad to multiple of 16 */
1034        p = PTR_ALIGN(pos, 8);
1035        if ((uintptr_t)p & 8)
1036                *p = 0;
1037}
1038EXPORT_SYMBOL(cxgb4_inline_tx_skb);
1039
1040static void *inline_tx_skb_header(const struct sk_buff *skb,
1041                                  const struct sge_txq *q,  void *pos,
1042                                  int length)
1043{
1044        u64 *p;
1045        int left = (void *)q->stat - pos;
1046
1047        if (likely(length <= left)) {
1048                memcpy(pos, skb->data, length);
1049                pos += length;
1050        } else {
1051                memcpy(pos, skb->data, left);
1052                memcpy(q->desc, skb->data + left, length - left);
1053                pos = (void *)q->desc + (length - left);
1054        }
1055        /* 0-pad to multiple of 16 */
1056        p = PTR_ALIGN(pos, 8);
1057        if ((uintptr_t)p & 8) {
1058                *p = 0;
1059                return p + 1;
1060        }
1061        return p;
1062}
1063
1064/*
1065 * Figure out what HW csum a packet wants and return the appropriate control
1066 * bits.
1067 */
1068static u64 hwcsum(enum chip_type chip, const struct sk_buff *skb)
1069{
1070        int csum_type;
1071        bool inner_hdr_csum = false;
1072        u16 proto, ver;
1073
1074        if (skb->encapsulation &&
1075            (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5))
1076                inner_hdr_csum = true;
1077
1078        if (inner_hdr_csum) {
1079                ver = inner_ip_hdr(skb)->version;
1080                proto = (ver == 4) ? inner_ip_hdr(skb)->protocol :
1081                        inner_ipv6_hdr(skb)->nexthdr;
1082        } else {
1083                ver = ip_hdr(skb)->version;
1084                proto = (ver == 4) ? ip_hdr(skb)->protocol :
1085                        ipv6_hdr(skb)->nexthdr;
1086        }
1087
1088        if (ver == 4) {
1089                if (proto == IPPROTO_TCP)
1090                        csum_type = TX_CSUM_TCPIP;
1091                else if (proto == IPPROTO_UDP)
1092                        csum_type = TX_CSUM_UDPIP;
1093                else {
1094nocsum:                 /*
1095                         * unknown protocol, disable HW csum
1096                         * and hope a bad packet is detected
1097                         */
1098                        return TXPKT_L4CSUM_DIS_F;
1099                }
1100        } else {
1101                /*
1102                 * this doesn't work with extension headers
1103                 */
1104                if (proto == IPPROTO_TCP)
1105                        csum_type = TX_CSUM_TCPIP6;
1106                else if (proto == IPPROTO_UDP)
1107                        csum_type = TX_CSUM_UDPIP6;
1108                else
1109                        goto nocsum;
1110        }
1111
1112        if (likely(csum_type >= TX_CSUM_TCPIP)) {
1113                int eth_hdr_len, l4_len;
1114                u64 hdr_len;
1115
1116                if (inner_hdr_csum) {
1117                        /* This allows checksum offload for all encapsulated
1118                         * packets like GRE etc..
1119                         */
1120                        l4_len = skb_inner_network_header_len(skb);
1121                        eth_hdr_len = skb_inner_network_offset(skb) - ETH_HLEN;
1122                } else {
1123                        l4_len = skb_network_header_len(skb);
1124                        eth_hdr_len = skb_network_offset(skb) - ETH_HLEN;
1125                }
1126                hdr_len = TXPKT_IPHDR_LEN_V(l4_len);
1127
1128                if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5)
1129                        hdr_len |= TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1130                else
1131                        hdr_len |= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len);
1132                return TXPKT_CSUM_TYPE_V(csum_type) | hdr_len;
1133        } else {
1134                int start = skb_transport_offset(skb);
1135
1136                return TXPKT_CSUM_TYPE_V(csum_type) |
1137                        TXPKT_CSUM_START_V(start) |
1138                        TXPKT_CSUM_LOC_V(start + skb->csum_offset);
1139        }
1140}
1141
1142static void eth_txq_stop(struct sge_eth_txq *q)
1143{
1144        netif_tx_stop_queue(q->txq);
1145        q->q.stops++;
1146}
1147
1148static inline void txq_advance(struct sge_txq *q, unsigned int n)
1149{
1150        q->in_use += n;
1151        q->pidx += n;
1152        if (q->pidx >= q->size)
1153                q->pidx -= q->size;
1154}
1155
1156/* Returns tunnel type if hardware supports offloading of the same.
1157 * It is called only for T5 and onwards.
1158 */
1159enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb)
1160{
1161        u8 l4_hdr = 0;
1162        enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
1163        struct port_info *pi = netdev_priv(skb->dev);
1164        struct adapter *adapter = pi->adapter;
1165
1166        if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1167            skb->inner_protocol != htons(ETH_P_TEB))
1168                return tnl_type;
1169
1170        switch (vlan_get_protocol(skb)) {
1171        case htons(ETH_P_IP):
1172                l4_hdr = ip_hdr(skb)->protocol;
1173                break;
1174        case htons(ETH_P_IPV6):
1175                l4_hdr = ipv6_hdr(skb)->nexthdr;
1176                break;
1177        default:
1178                return tnl_type;
1179        }
1180
1181        switch (l4_hdr) {
1182        case IPPROTO_UDP:
1183                if (adapter->vxlan_port == udp_hdr(skb)->dest)
1184                        tnl_type = TX_TNL_TYPE_VXLAN;
1185                else if (adapter->geneve_port == udp_hdr(skb)->dest)
1186                        tnl_type = TX_TNL_TYPE_GENEVE;
1187                break;
1188        default:
1189                return tnl_type;
1190        }
1191
1192        return tnl_type;
1193}
1194
1195static inline void t6_fill_tnl_lso(struct sk_buff *skb,
1196                                   struct cpl_tx_tnl_lso *tnl_lso,
1197                                   enum cpl_tx_tnl_lso_type tnl_type)
1198{
1199        u32 val;
1200        int in_eth_xtra_len;
1201        int l3hdr_len = skb_network_header_len(skb);
1202        int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1203        const struct skb_shared_info *ssi = skb_shinfo(skb);
1204        bool v6 = (ip_hdr(skb)->version == 6);
1205
1206        val = CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO) |
1207              CPL_TX_TNL_LSO_FIRST_F |
1208              CPL_TX_TNL_LSO_LAST_F |
1209              (v6 ? CPL_TX_TNL_LSO_IPV6OUT_F : 0) |
1210              CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len / 4) |
1211              CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len / 4) |
1212              (v6 ? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F) |
1213              CPL_TX_TNL_LSO_IPLENSETOUT_F |
1214              (v6 ? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F);
1215        tnl_lso->op_to_IpIdSplitOut = htonl(val);
1216
1217        tnl_lso->IpIdOffsetOut = 0;
1218
1219        /* Get the tunnel header length */
1220        val = skb_inner_mac_header(skb) - skb_mac_header(skb);
1221        in_eth_xtra_len = skb_inner_network_header(skb) -
1222                          skb_inner_mac_header(skb) - ETH_HLEN;
1223
1224        switch (tnl_type) {
1225        case TX_TNL_TYPE_VXLAN:
1226        case TX_TNL_TYPE_GENEVE:
1227                tnl_lso->UdpLenSetOut_to_TnlHdrLen =
1228                        htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F |
1229                        CPL_TX_TNL_LSO_UDPLENSETOUT_F);
1230                break;
1231        default:
1232                tnl_lso->UdpLenSetOut_to_TnlHdrLen = 0;
1233                break;
1234        }
1235
1236        tnl_lso->UdpLenSetOut_to_TnlHdrLen |=
1237                 htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val) |
1238                       CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type));
1239
1240        tnl_lso->r1 = 0;
1241
1242        val = CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len / 4) |
1243              CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb)->version == 6) |
1244              CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb) / 4) |
1245              CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb) / 4);
1246        tnl_lso->Flow_to_TcpHdrLen = htonl(val);
1247
1248        tnl_lso->IpIdOffset = htons(0);
1249
1250        tnl_lso->IpIdSplit_to_Mss = htons(CPL_TX_TNL_LSO_MSS_V(ssi->gso_size));
1251        tnl_lso->TCPSeqOffset = htonl(0);
1252        tnl_lso->EthLenOffset_Size = htonl(CPL_TX_TNL_LSO_SIZE_V(skb->len));
1253}
1254
1255/**
1256 *      cxgb4_eth_xmit - add a packet to an Ethernet Tx queue
1257 *      @skb: the packet
1258 *      @dev: the egress net device
1259 *
1260 *      Add a packet to an SGE Ethernet Tx queue.  Runs with softirqs disabled.
1261 */
1262static netdev_tx_t cxgb4_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1263{
1264        u32 wr_mid, ctrl0, op;
1265        u64 cntrl = 0;
1266        u64 *end, *sgl;
1267        int qidx, credits;
1268        unsigned int flits, ndesc;
1269        struct adapter *adap;
1270        struct sge_eth_txq *q;
1271        const struct port_info *pi;
1272        struct fw_eth_tx_pkt_wr *wr;
1273        struct cpl_tx_pkt_core *cpl;
1274        const struct skb_shared_info *ssi;
1275        dma_addr_t addr[MAX_SKB_FRAGS + 1];
1276        bool immediate = false;
1277        int len, max_pkt_len;
1278        bool ptp_enabled = is_ptp_enabled(skb, dev);
1279        unsigned int chip_ver;
1280        enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
1281
1282
1283        /*
1284         * The chip min packet length is 10 octets but play safe and reject
1285         * anything shorter than an Ethernet header.
1286         */
1287        if (unlikely(skb->len < ETH_HLEN)) {
1288out_free:       dev_kfree_skb_any(skb);
1289                return NETDEV_TX_OK;
1290        }
1291
1292        /* Discard the packet if the length is greater than mtu */
1293        max_pkt_len = ETH_HLEN + dev->mtu;
1294        if (skb_vlan_tagged(skb)) 
1295                max_pkt_len += VLAN_HLEN;
1296        if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1297                goto out_free;
1298
1299        pi = netdev_priv(dev);
1300        adap = pi->adapter;
1301        ssi = skb_shinfo(skb);
1302#ifdef CONFIG_CHELSIO_IPSEC_INLINE
1303        if (xfrm_offload(skb) && !ssi->gso_size)
1304                return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev);
1305#endif /* CHELSIO_IPSEC_INLINE */
1306
1307        qidx = skb_get_queue_mapping(skb);
1308        if (ptp_enabled) {
1309                spin_lock(&adap->ptp_lock);
1310                if (!(adap->ptp_tx_skb)) {
1311                        skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1312                        adap->ptp_tx_skb = skb_get(skb);
1313                } else {
1314                        spin_unlock(&adap->ptp_lock);
1315                        goto out_free;
1316                }
1317                q = &adap->sge.ptptxq;
1318        } else {
1319                q = &adap->sge.ethtxq[qidx + pi->first_qset];
1320        }
1321        skb_tx_timestamp(skb);
1322
1323        cxgb4_reclaim_completed_tx(adap, &q->q, true);
1324
1325        chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
1326        flits = calc_tx_flits(skb, chip_ver);
1327        ndesc = flits_to_desc(flits);
1328        credits = txq_avail(&q->q) - ndesc;
1329
1330        if (unlikely(credits < 0)) {
1331                eth_txq_stop(q);
1332                dev_err(adap->pdev_dev,
1333                        "%s: Tx ring %u full while queue awake!\n",
1334                        dev->name, qidx);
1335                if (ptp_enabled)
1336                        spin_unlock(&adap->ptp_lock);
1337                return NETDEV_TX_BUSY;
1338        }
1339
1340        if (is_eth_imm(skb, chip_ver))
1341                immediate = true;
1342
1343        if (skb->encapsulation && chip_ver > CHELSIO_T5)
1344                tnl_type = cxgb_encap_offload_supported(skb);
1345
1346        if (!immediate &&
1347            unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
1348                q->mapping_err++;
1349                if (ptp_enabled)
1350                        spin_unlock(&adap->ptp_lock);
1351                goto out_free;
1352        }
1353
1354        wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1355        if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1356                eth_txq_stop(q);
1357                wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1358        }
1359
1360        wr = (void *)&q->q.desc[q->q.pidx];
1361        wr->equiq_to_len16 = htonl(wr_mid);
1362        wr->r3 = cpu_to_be64(0);
1363        end = (u64 *)wr + flits;
1364
1365        len = immediate ? skb->len : 0;
1366        len += sizeof(*cpl);
1367        if (ssi->gso_size) {
1368                struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1369                bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1370                int l3hdr_len = skb_network_header_len(skb);
1371                int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1372                struct cpl_tx_tnl_lso *tnl_lso = (void *)(wr + 1);
1373
1374                if (tnl_type)
1375                        len += sizeof(*tnl_lso);
1376                else
1377                        len += sizeof(*lso);
1378
1379                wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1380                                       FW_WR_IMMDLEN_V(len));
1381                if (tnl_type) {
1382                        struct iphdr *iph = ip_hdr(skb);
1383
1384                        t6_fill_tnl_lso(skb, tnl_lso, tnl_type);
1385                        cpl = (void *)(tnl_lso + 1);
1386                        /* Driver is expected to compute partial checksum that
1387                         * does not include the IP Total Length.
1388                         */
1389                        if (iph->version == 4) {
1390                                iph->check = 0;
1391                                iph->tot_len = 0;
1392                                iph->check = (u16)(~ip_fast_csum((u8 *)iph,
1393                                                                 iph->ihl));
1394                        }
1395                        if (skb->ip_summed == CHECKSUM_PARTIAL)
1396                                cntrl = hwcsum(adap->params.chip, skb);
1397                } else {
1398                        lso->lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1399                                        LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
1400                                        LSO_IPV6_V(v6) |
1401                                        LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1402                                        LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1403                                        LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1404                        lso->ipid_ofst = htons(0);
1405                        lso->mss = htons(ssi->gso_size);
1406                        lso->seqno_offset = htonl(0);
1407                        if (is_t4(adap->params.chip))
1408                                lso->len = htonl(skb->len);
1409                        else
1410                                lso->len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
1411                        cpl = (void *)(lso + 1);
1412
1413                        if (CHELSIO_CHIP_VERSION(adap->params.chip)
1414                            <= CHELSIO_T5)
1415                                cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1416                        else
1417                                cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1418
1419                        cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1420                                 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1421                                 TXPKT_IPHDR_LEN_V(l3hdr_len);
1422                }
1423                sgl = (u64 *)(cpl + 1); /* sgl start here */
1424                if (unlikely((u8 *)sgl >= (u8 *)q->q.stat)) {
1425                        /* If current position is already at the end of the
1426                         * txq, reset the current to point to start of the queue
1427                         * and update the end ptr as well.
1428                         */
1429                        if (sgl == (u64 *)q->q.stat) {
1430                                int left = (u8 *)end - (u8 *)q->q.stat;
1431
1432                                end = (void *)q->q.desc + left;
1433                                sgl = (void *)q->q.desc;
1434                        }
1435                }
1436                q->tso++;
1437                q->tx_cso += ssi->gso_segs;
1438        } else {
1439                if (ptp_enabled)
1440                        op = FW_PTP_TX_PKT_WR;
1441                else
1442                        op = FW_ETH_TX_PKT_WR;
1443                wr->op_immdlen = htonl(FW_WR_OP_V(op) |
1444                                       FW_WR_IMMDLEN_V(len));
1445                cpl = (void *)(wr + 1);
1446                sgl = (u64 *)(cpl + 1);
1447                if (skb->ip_summed == CHECKSUM_PARTIAL) {
1448                        cntrl = hwcsum(adap->params.chip, skb) |
1449                                TXPKT_IPCSUM_DIS_F;
1450                        q->tx_cso++;
1451                } else
1452                        cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1453        }
1454
1455        if (skb_vlan_tag_present(skb)) {
1456                q->vlan_ins++;
1457                cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1458        }
1459
1460        ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
1461                TXPKT_PF_V(adap->pf);
1462        if (ptp_enabled)
1463                ctrl0 |= TXPKT_TSTAMP_F;
1464#ifdef CONFIG_CHELSIO_T4_DCB
1465        if (is_t4(adap->params.chip))
1466                ctrl0 |= TXPKT_OVLAN_IDX_V(q->dcb_prio);
1467        else
1468                ctrl0 |= TXPKT_T5_OVLAN_IDX_V(q->dcb_prio);
1469#endif
1470        cpl->ctrl0 = htonl(ctrl0);
1471        cpl->pack = htons(0);
1472        cpl->len = htons(skb->len);
1473        cpl->ctrl1 = cpu_to_be64(cntrl);
1474
1475        if (immediate) {
1476                cxgb4_inline_tx_skb(skb, &q->q, sgl);
1477                dev_consume_skb_any(skb);
1478        } else {
1479                int last_desc;
1480
1481                cxgb4_write_sgl(skb, &q->q, (void *)sgl, end, 0, addr);
1482                skb_orphan(skb);
1483
1484                last_desc = q->q.pidx + ndesc - 1;
1485                if (last_desc >= q->q.size)
1486                        last_desc -= q->q.size;
1487                q->q.sdesc[last_desc].skb = skb;
1488                q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
1489        }
1490
1491        txq_advance(&q->q, ndesc);
1492
1493        cxgb4_ring_tx_db(adap, &q->q, ndesc);
1494        if (ptp_enabled)
1495                spin_unlock(&adap->ptp_lock);
1496        return NETDEV_TX_OK;
1497}
1498
1499/* Constants ... */
1500enum {
1501        /* Egress Queue sizes, producer and consumer indices are all in units
1502         * of Egress Context Units bytes.  Note that as far as the hardware is
1503         * concerned, the free list is an Egress Queue (the host produces free
1504         * buffers which the hardware consumes) and free list entries are
1505         * 64-bit PCI DMA addresses.
1506         */
1507        EQ_UNIT = SGE_EQ_IDXSIZE,
1508        FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
1509        TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
1510
1511        T4VF_ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1512                               sizeof(struct cpl_tx_pkt_lso_core) +
1513                               sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64),
1514};
1515
1516/**
1517 *      t4vf_is_eth_imm - can an Ethernet packet be sent as immediate data?
1518 *      @skb: the packet
1519 *
1520 *      Returns whether an Ethernet packet is small enough to fit completely as
1521 *      immediate data.
1522 */
1523static inline int t4vf_is_eth_imm(const struct sk_buff *skb)
1524{
1525        /* The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request
1526         * which does not accommodate immediate data.  We could dike out all
1527         * of the support code for immediate data but that would tie our hands
1528         * too much if we ever want to enhace the firmware.  It would also
1529         * create more differences between the PF and VF Drivers.
1530         */
1531        return false;
1532}
1533
1534/**
1535 *      t4vf_calc_tx_flits - calculate the number of flits for a packet TX WR
1536 *      @skb: the packet
1537 *
1538 *      Returns the number of flits needed for a TX Work Request for the
1539 *      given Ethernet packet, including the needed WR and CPL headers.
1540 */
1541static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff *skb)
1542{
1543        unsigned int flits;
1544
1545        /* If the skb is small enough, we can pump it out as a work request
1546         * with only immediate data.  In that case we just have to have the
1547         * TX Packet header plus the skb data in the Work Request.
1548         */
1549        if (t4vf_is_eth_imm(skb))
1550                return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt),
1551                                    sizeof(__be64));
1552
1553        /* Otherwise, we're going to have to construct a Scatter gather list
1554         * of the skb body and fragments.  We also include the flits necessary
1555         * for the TX Packet Work Request and CPL.  We always have a firmware
1556         * Write Header (incorporated as part of the cpl_tx_pkt_lso and
1557         * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
1558         * message or, if we're doing a Large Send Offload, an LSO CPL message
1559         * with an embedded TX Packet Write CPL message.
1560         */
1561        flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
1562        if (skb_shinfo(skb)->gso_size)
1563                flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1564                          sizeof(struct cpl_tx_pkt_lso_core) +
1565                          sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
1566        else
1567                flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) +
1568                          sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
1569        return flits;
1570}
1571
1572/**
1573 *      cxgb4_vf_eth_xmit - add a packet to an Ethernet TX queue
1574 *      @skb: the packet
1575 *      @dev: the egress net device
1576 *
1577 *      Add a packet to an SGE Ethernet TX queue.  Runs with softirqs disabled.
1578 */
1579static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb,
1580                                     struct net_device *dev)
1581{
1582        dma_addr_t addr[MAX_SKB_FRAGS + 1];
1583        const struct skb_shared_info *ssi;
1584        struct fw_eth_tx_pkt_vm_wr *wr;
1585        int qidx, credits, max_pkt_len;
1586        struct cpl_tx_pkt_core *cpl;
1587        const struct port_info *pi;
1588        unsigned int flits, ndesc;
1589        struct sge_eth_txq *txq;
1590        struct adapter *adapter;
1591        u64 cntrl, *end;
1592        u32 wr_mid;
1593        const size_t fw_hdr_copy_len = sizeof(wr->ethmacdst) +
1594                                       sizeof(wr->ethmacsrc) +
1595                                       sizeof(wr->ethtype) +
1596                                       sizeof(wr->vlantci);
1597
1598        /* The chip minimum packet length is 10 octets but the firmware
1599         * command that we are using requires that we copy the Ethernet header
1600         * (including the VLAN tag) into the header so we reject anything
1601         * smaller than that ...
1602         */
1603        if (unlikely(skb->len < fw_hdr_copy_len))
1604                goto out_free;
1605
1606        /* Discard the packet if the length is greater than mtu */
1607        max_pkt_len = ETH_HLEN + dev->mtu;
1608        if (skb_vlan_tag_present(skb))
1609                max_pkt_len += VLAN_HLEN;
1610        if (!skb_shinfo(skb)->gso_size && (unlikely(skb->len > max_pkt_len)))
1611                goto out_free;
1612
1613        /* Figure out which TX Queue we're going to use. */
1614        pi = netdev_priv(dev);
1615        adapter = pi->adapter;
1616        qidx = skb_get_queue_mapping(skb);
1617        WARN_ON(qidx >= pi->nqsets);
1618        txq = &adapter->sge.ethtxq[pi->first_qset + qidx];
1619
1620        /* Take this opportunity to reclaim any TX Descriptors whose DMA
1621         * transfers have completed.
1622         */
1623        cxgb4_reclaim_completed_tx(adapter, &txq->q, true);
1624
1625        /* Calculate the number of flits and TX Descriptors we're going to
1626         * need along with how many TX Descriptors will be left over after
1627         * we inject our Work Request.
1628         */
1629        flits = t4vf_calc_tx_flits(skb);
1630        ndesc = flits_to_desc(flits);
1631        credits = txq_avail(&txq->q) - ndesc;
1632
1633        if (unlikely(credits < 0)) {
1634                /* Not enough room for this packet's Work Request.  Stop the
1635                 * TX Queue and return a "busy" condition.  The queue will get
1636                 * started later on when the firmware informs us that space
1637                 * has opened up.
1638                 */
1639                eth_txq_stop(txq);
1640                dev_err(adapter->pdev_dev,
1641                        "%s: TX ring %u full while queue awake!\n",
1642                        dev->name, qidx);
1643                return NETDEV_TX_BUSY;
1644        }
1645
1646        if (!t4vf_is_eth_imm(skb) &&
1647            unlikely(cxgb4_map_skb(adapter->pdev_dev, skb, addr) < 0)) {
1648                /* We need to map the skb into PCI DMA space (because it can't
1649                 * be in-lined directly into the Work Request) and the mapping
1650                 * operation failed.  Record the error and drop the packet.
1651                 */
1652                txq->mapping_err++;
1653                goto out_free;
1654        }
1655
1656        wr_mid = FW_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
1657        if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1658                /* After we're done injecting the Work Request for this
1659                 * packet, we'll be below our "stop threshold" so stop the TX
1660                 * Queue now and schedule a request for an SGE Egress Queue
1661                 * Update message.  The queue will get started later on when
1662                 * the firmware processes this Work Request and sends us an
1663                 * Egress Queue Status Update message indicating that space
1664                 * has opened up.
1665                 */
1666                eth_txq_stop(txq);
1667                wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1668        }
1669
1670        /* Start filling in our Work Request.  Note that we do _not_ handle
1671         * the WR Header wrapping around the TX Descriptor Ring.  If our
1672         * maximum header size ever exceeds one TX Descriptor, we'll need to
1673         * do something else here.
1674         */
1675        WARN_ON(DIV_ROUND_UP(T4VF_ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1);
1676        wr = (void *)&txq->q.desc[txq->q.pidx];
1677        wr->equiq_to_len16 = cpu_to_be32(wr_mid);
1678        wr->r3[0] = cpu_to_be32(0);
1679        wr->r3[1] = cpu_to_be32(0);
1680        skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len);
1681        end = (u64 *)wr + flits;
1682
1683        /* If this is a Large Send Offload packet we'll put in an LSO CPL
1684         * message with an encapsulated TX Packet CPL message.  Otherwise we
1685         * just use a TX Packet CPL message.
1686         */
1687        ssi = skb_shinfo(skb);
1688        if (ssi->gso_size) {
1689                struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
1690                bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
1691                int l3hdr_len = skb_network_header_len(skb);
1692                int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
1693
1694                wr->op_immdlen =
1695                        cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1696                                    FW_WR_IMMDLEN_V(sizeof(*lso) +
1697                                                    sizeof(*cpl)));
1698                 /* Fill in the LSO CPL message. */
1699                lso->lso_ctrl =
1700                        cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
1701                                    LSO_FIRST_SLICE_F |
1702                                    LSO_LAST_SLICE_F |
1703                                    LSO_IPV6_V(v6) |
1704                                    LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
1705                                    LSO_IPHDR_LEN_V(l3hdr_len / 4) |
1706                                    LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
1707                lso->ipid_ofst = cpu_to_be16(0);
1708                lso->mss = cpu_to_be16(ssi->gso_size);
1709                lso->seqno_offset = cpu_to_be32(0);
1710                if (is_t4(adapter->params.chip))
1711                        lso->len = cpu_to_be32(skb->len);
1712                else
1713                        lso->len = cpu_to_be32(LSO_T5_XFER_SIZE_V(skb->len));
1714
1715                /* Set up TX Packet CPL pointer, control word and perform
1716                 * accounting.
1717                 */
1718                cpl = (void *)(lso + 1);
1719
1720                if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
1721                        cntrl = TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1722                else
1723                        cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
1724
1725                cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
1726                                           TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
1727                         TXPKT_IPHDR_LEN_V(l3hdr_len);
1728                txq->tso++;
1729                txq->tx_cso += ssi->gso_segs;
1730        } else {
1731                int len;
1732
1733                len = (t4vf_is_eth_imm(skb)
1734                       ? skb->len + sizeof(*cpl)
1735                       : sizeof(*cpl));
1736                wr->op_immdlen =
1737                        cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR) |
1738                                    FW_WR_IMMDLEN_V(len));
1739
1740                /* Set up TX Packet CPL pointer, control word and perform
1741                 * accounting.
1742                 */
1743                cpl = (void *)(wr + 1);
1744                if (skb->ip_summed == CHECKSUM_PARTIAL) {
1745                        cntrl = hwcsum(adapter->params.chip, skb) |
1746                                TXPKT_IPCSUM_DIS_F;
1747                        txq->tx_cso++;
1748                } else {
1749                        cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
1750                }
1751        }
1752
1753        /* If there's a VLAN tag present, add that to the list of things to
1754         * do in this Work Request.
1755         */
1756        if (skb_vlan_tag_present(skb)) {
1757                txq->vlan_ins++;
1758                cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
1759        }
1760
1761         /* Fill in the TX Packet CPL message header. */
1762        cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT) |
1763                                 TXPKT_INTF_V(pi->port_id) |
1764                                 TXPKT_PF_V(0));
1765        cpl->pack = cpu_to_be16(0);
1766        cpl->len = cpu_to_be16(skb->len);
1767        cpl->ctrl1 = cpu_to_be64(cntrl);
1768
1769        /* Fill in the body of the TX Packet CPL message with either in-lined
1770         * data or a Scatter/Gather List.
1771         */
1772        if (t4vf_is_eth_imm(skb)) {
1773                /* In-line the packet's data and free the skb since we don't
1774                 * need it any longer.
1775                 */
1776                cxgb4_inline_tx_skb(skb, &txq->q, cpl + 1);
1777                dev_consume_skb_any(skb);
1778        } else {
1779                /* Write the skb's Scatter/Gather list into the TX Packet CPL
1780                 * message and retain a pointer to the skb so we can free it
1781                 * later when its DMA completes.  (We store the skb pointer
1782                 * in the Software Descriptor corresponding to the last TX
1783                 * Descriptor used by the Work Request.)
1784                 *
1785                 * The retained skb will be freed when the corresponding TX
1786                 * Descriptors are reclaimed after their DMAs complete.
1787                 * However, this could take quite a while since, in general,
1788                 * the hardware is set up to be lazy about sending DMA
1789                 * completion notifications to us and we mostly perform TX
1790                 * reclaims in the transmit routine.
1791                 *
1792                 * This is good for performamce but means that we rely on new
1793                 * TX packets arriving to run the destructors of completed
1794                 * packets, which open up space in their sockets' send queues.
1795                 * Sometimes we do not get such new packets causing TX to
1796                 * stall.  A single UDP transmitter is a good example of this
1797                 * situation.  We have a clean up timer that periodically
1798                 * reclaims completed packets but it doesn't run often enough
1799                 * (nor do we want it to) to prevent lengthy stalls.  A
1800                 * solution to this problem is to run the destructor early,
1801                 * after the packet is queued but before it's DMAd.  A con is
1802                 * that we lie to socket memory accounting, but the amount of
1803                 * extra memory is reasonable (limited by the number of TX
1804                 * descriptors), the packets do actually get freed quickly by
1805                 * new packets almost always, and for protocols like TCP that
1806                 * wait for acks to really free up the data the extra memory
1807                 * is even less.  On the positive side we run the destructors
1808                 * on the sending CPU rather than on a potentially different
1809                 * completing CPU, usually a good thing.
1810                 *
1811                 * Run the destructor before telling the DMA engine about the
1812                 * packet to make sure it doesn't complete and get freed
1813                 * prematurely.
1814                 */
1815                struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1);
1816                struct sge_txq *tq = &txq->q;
1817                int last_desc;
1818
1819                /* If the Work Request header was an exact multiple of our TX
1820                 * Descriptor length, then it's possible that the starting SGL
1821                 * pointer lines up exactly with the end of our TX Descriptor
1822                 * ring.  If that's the case, wrap around to the beginning
1823                 * here ...
1824                 */
1825                if (unlikely((void *)sgl == (void *)tq->stat)) {
1826                        sgl = (void *)tq->desc;
1827                        end = (void *)((void *)tq->desc +
1828                                       ((void *)end - (void *)tq->stat));
1829                }
1830
1831                cxgb4_write_sgl(skb, tq, sgl, end, 0, addr);
1832                skb_orphan(skb);
1833
1834                last_desc = tq->pidx + ndesc - 1;
1835                if (last_desc >= tq->size)
1836                        last_desc -= tq->size;
1837                tq->sdesc[last_desc].skb = skb;
1838                tq->sdesc[last_desc].sgl = sgl;
1839        }
1840
1841        /* Advance our internal TX Queue state, tell the hardware about
1842         * the new TX descriptors and return success.
1843         */
1844        txq_advance(&txq->q, ndesc);
1845
1846        cxgb4_ring_tx_db(adapter, &txq->q, ndesc);
1847        return NETDEV_TX_OK;
1848
1849out_free:
1850        /* An error of some sort happened.  Free the TX skb and tell the
1851         * OS that we've "dealt" with the packet ...
1852         */
1853        dev_kfree_skb_any(skb);
1854        return NETDEV_TX_OK;
1855}
1856
1857netdev_tx_t t4_start_xmit(struct sk_buff *skb, struct net_device *dev)
1858{
1859        struct port_info *pi = netdev_priv(dev);
1860
1861        if (unlikely(pi->eth_flags & PRIV_FLAG_PORT_TX_VM))
1862                return cxgb4_vf_eth_xmit(skb, dev);
1863
1864        return cxgb4_eth_xmit(skb, dev);
1865}
1866
1867/**
1868 *      reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1869 *      @q: the SGE control Tx queue
1870 *
1871 *      This is a variant of cxgb4_reclaim_completed_tx() that is used
1872 *      for Tx queues that send only immediate data (presently just
1873 *      the control queues) and thus do not have any sk_buffs to release.
1874 */
1875static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1876{
1877        int hw_cidx = ntohs(READ_ONCE(q->stat->cidx));
1878        int reclaim = hw_cidx - q->cidx;
1879
1880        if (reclaim < 0)
1881                reclaim += q->size;
1882
1883        q->in_use -= reclaim;
1884        q->cidx = hw_cidx;
1885}
1886
1887/**
1888 *      is_imm - check whether a packet can be sent as immediate data
1889 *      @skb: the packet
1890 *
1891 *      Returns true if a packet can be sent as a WR with immediate data.
1892 */
1893static inline int is_imm(const struct sk_buff *skb)
1894{
1895        return skb->len <= MAX_CTRL_WR_LEN;
1896}
1897
1898/**
1899 *      ctrlq_check_stop - check if a control queue is full and should stop
1900 *      @q: the queue
1901 *      @wr: most recent WR written to the queue
1902 *
1903 *      Check if a control queue has become full and should be stopped.
1904 *      We clean up control queue descriptors very lazily, only when we are out.
1905 *      If the queue is still full after reclaiming any completed descriptors
1906 *      we suspend it and have the last WR wake it up.
1907 */
1908static void ctrlq_check_stop(struct sge_ctrl_txq *q, struct fw_wr_hdr *wr)
1909{
1910        reclaim_completed_tx_imm(&q->q);
1911        if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
1912                wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
1913                q->q.stops++;
1914                q->full = 1;
1915        }
1916}
1917
1918/**
1919 *      ctrl_xmit - send a packet through an SGE control Tx queue
1920 *      @q: the control queue
1921 *      @skb: the packet
1922 *
1923 *      Send a packet through an SGE control Tx queue.  Packets sent through
1924 *      a control queue must fit entirely as immediate data.
1925 */
1926static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb)
1927{
1928        unsigned int ndesc;
1929        struct fw_wr_hdr *wr;
1930
1931        if (unlikely(!is_imm(skb))) {
1932                WARN_ON(1);
1933                dev_kfree_skb(skb);
1934                return NET_XMIT_DROP;
1935        }
1936
1937        ndesc = DIV_ROUND_UP(skb->len, sizeof(struct tx_desc));
1938        spin_lock(&q->sendq.lock);
1939
1940        if (unlikely(q->full)) {
1941                skb->priority = ndesc;                  /* save for restart */
1942                __skb_queue_tail(&q->sendq, skb);
1943                spin_unlock(&q->sendq.lock);
1944                return NET_XMIT_CN;
1945        }
1946
1947        wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
1948        cxgb4_inline_tx_skb(skb, &q->q, wr);
1949
1950        txq_advance(&q->q, ndesc);
1951        if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES))
1952                ctrlq_check_stop(q, wr);
1953
1954        cxgb4_ring_tx_db(q->adap, &q->q, ndesc);
1955        spin_unlock(&q->sendq.lock);
1956
1957        kfree_skb(skb);
1958        return NET_XMIT_SUCCESS;
1959}
1960
1961/**
1962 *      restart_ctrlq - restart a suspended control queue
1963 *      @data: the control queue to restart
1964 *
1965 *      Resumes transmission on a suspended Tx control queue.
1966 */
1967static void restart_ctrlq(unsigned long data)
1968{
1969        struct sk_buff *skb;
1970        unsigned int written = 0;
1971        struct sge_ctrl_txq *q = (struct sge_ctrl_txq *)data;
1972
1973        spin_lock(&q->sendq.lock);
1974        reclaim_completed_tx_imm(&q->q);
1975        BUG_ON(txq_avail(&q->q) < TXQ_STOP_THRES);  /* q should be empty */
1976
1977        while ((skb = __skb_dequeue(&q->sendq)) != NULL) {
1978                struct fw_wr_hdr *wr;
1979                unsigned int ndesc = skb->priority;     /* previously saved */
1980
1981                written += ndesc;
1982                /* Write descriptors and free skbs outside the lock to limit
1983                 * wait times.  q->full is still set so new skbs will be queued.
1984                 */
1985                wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
1986                txq_advance(&q->q, ndesc);
1987                spin_unlock(&q->sendq.lock);
1988
1989                cxgb4_inline_tx_skb(skb, &q->q, wr);
1990                kfree_skb(skb);
1991
1992                if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
1993                        unsigned long old = q->q.stops;
1994
1995                        ctrlq_check_stop(q, wr);
1996                        if (q->q.stops != old) {          /* suspended anew */
1997                                spin_lock(&q->sendq.lock);
1998                                goto ringdb;
1999                        }
2000                }
2001                if (written > 16) {
2002                        cxgb4_ring_tx_db(q->adap, &q->q, written);
2003                        written = 0;
2004                }
2005                spin_lock(&q->sendq.lock);
2006        }
2007        q->full = 0;
2008ringdb:
2009        if (written)
2010                cxgb4_ring_tx_db(q->adap, &q->q, written);
2011        spin_unlock(&q->sendq.lock);
2012}
2013
2014/**
2015 *      t4_mgmt_tx - send a management message
2016 *      @adap: the adapter
2017 *      @skb: the packet containing the management message
2018 *
2019 *      Send a management message through control queue 0.
2020 */
2021int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
2022{
2023        int ret;
2024
2025        local_bh_disable();
2026        ret = ctrl_xmit(&adap->sge.ctrlq[0], skb);
2027        local_bh_enable();
2028        return ret;
2029}
2030
2031/**
2032 *      is_ofld_imm - check whether a packet can be sent as immediate data
2033 *      @skb: the packet
2034 *
2035 *      Returns true if a packet can be sent as an offload WR with immediate
2036 *      data.  We currently use the same limit as for Ethernet packets.
2037 */
2038static inline int is_ofld_imm(const struct sk_buff *skb)
2039{
2040        struct work_request_hdr *req = (struct work_request_hdr *)skb->data;
2041        unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi));
2042
2043        if (opcode == FW_CRYPTO_LOOKASIDE_WR)
2044                return skb->len <= SGE_MAX_WR_LEN;
2045        else
2046                return skb->len <= MAX_IMM_TX_PKT_LEN;
2047}
2048
2049/**
2050 *      calc_tx_flits_ofld - calculate # of flits for an offload packet
2051 *      @skb: the packet
2052 *
2053 *      Returns the number of flits needed for the given offload packet.
2054 *      These packets are already fully constructed and no additional headers
2055 *      will be added.
2056 */
2057static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
2058{
2059        unsigned int flits, cnt;
2060
2061        if (is_ofld_imm(skb))
2062                return DIV_ROUND_UP(skb->len, 8);
2063
2064        flits = skb_transport_offset(skb) / 8U;   /* headers */
2065        cnt = skb_shinfo(skb)->nr_frags;
2066        if (skb_tail_pointer(skb) != skb_transport_header(skb))
2067                cnt++;
2068        return flits + sgl_len(cnt);
2069}
2070
2071/**
2072 *      txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion
2073 *      @adap: the adapter
2074 *      @q: the queue to stop
2075 *
2076 *      Mark a Tx queue stopped due to I/O MMU exhaustion and resulting
2077 *      inability to map packets.  A periodic timer attempts to restart
2078 *      queues so marked.
2079 */
2080static void txq_stop_maperr(struct sge_uld_txq *q)
2081{
2082        q->mapping_err++;
2083        q->q.stops++;
2084        set_bit(q->q.cntxt_id - q->adap->sge.egr_start,
2085                q->adap->sge.txq_maperr);
2086}
2087
2088/**
2089 *      ofldtxq_stop - stop an offload Tx queue that has become full
2090 *      @q: the queue to stop
2091 *      @wr: the Work Request causing the queue to become full
2092 *
2093 *      Stops an offload Tx queue that has become full and modifies the packet
2094 *      being written to request a wakeup.
2095 */
2096static void ofldtxq_stop(struct sge_uld_txq *q, struct fw_wr_hdr *wr)
2097{
2098        wr->lo |= htonl(FW_WR_EQUEQ_F | FW_WR_EQUIQ_F);
2099        q->q.stops++;
2100        q->full = 1;
2101}
2102
2103/**
2104 *      service_ofldq - service/restart a suspended offload queue
2105 *      @q: the offload queue
2106 *
2107 *      Services an offload Tx queue by moving packets from its Pending Send
2108 *      Queue to the Hardware TX ring.  The function starts and ends with the
2109 *      Send Queue locked, but drops the lock while putting the skb at the
2110 *      head of the Send Queue onto the Hardware TX Ring.  Dropping the lock
2111 *      allows more skbs to be added to the Send Queue by other threads.
2112 *      The packet being processed at the head of the Pending Send Queue is
2113 *      left on the queue in case we experience DMA Mapping errors, etc.
2114 *      and need to give up and restart later.
2115 *
2116 *      service_ofldq() can be thought of as a task which opportunistically
2117 *      uses other threads execution contexts.  We use the Offload Queue
2118 *      boolean "service_ofldq_running" to make sure that only one instance
2119 *      is ever running at a time ...
2120 */
2121static void service_ofldq(struct sge_uld_txq *q)
2122{
2123        u64 *pos, *before, *end;
2124        int credits;
2125        struct sk_buff *skb;
2126        struct sge_txq *txq;
2127        unsigned int left;
2128        unsigned int written = 0;
2129        unsigned int flits, ndesc;
2130
2131        /* If another thread is currently in service_ofldq() processing the
2132         * Pending Send Queue then there's nothing to do. Otherwise, flag
2133         * that we're doing the work and continue.  Examining/modifying
2134         * the Offload Queue boolean "service_ofldq_running" must be done
2135         * while holding the Pending Send Queue Lock.
2136         */
2137        if (q->service_ofldq_running)
2138                return;
2139        q->service_ofldq_running = true;
2140
2141        while ((skb = skb_peek(&q->sendq)) != NULL && !q->full) {
2142                /* We drop the lock while we're working with the skb at the
2143                 * head of the Pending Send Queue.  This allows more skbs to
2144                 * be added to the Pending Send Queue while we're working on
2145                 * this one.  We don't need to lock to guard the TX Ring
2146                 * updates because only one thread of execution is ever
2147                 * allowed into service_ofldq() at a time.
2148                 */
2149                spin_unlock(&q->sendq.lock);
2150
2151                cxgb4_reclaim_completed_tx(q->adap, &q->q, false);
2152
2153                flits = skb->priority;                /* previously saved */
2154                ndesc = flits_to_desc(flits);
2155                credits = txq_avail(&q->q) - ndesc;
2156                BUG_ON(credits < 0);
2157                if (unlikely(credits < TXQ_STOP_THRES))
2158                        ofldtxq_stop(q, (struct fw_wr_hdr *)skb->data);
2159
2160                pos = (u64 *)&q->q.desc[q->q.pidx];
2161                if (is_ofld_imm(skb))
2162                        cxgb4_inline_tx_skb(skb, &q->q, pos);
2163                else if (cxgb4_map_skb(q->adap->pdev_dev, skb,
2164                                       (dma_addr_t *)skb->head)) {
2165                        txq_stop_maperr(q);
2166                        spin_lock(&q->sendq.lock);
2167                        break;
2168                } else {
2169                        int last_desc, hdr_len = skb_transport_offset(skb);
2170
2171                        /* The WR headers  may not fit within one descriptor.
2172                         * So we need to deal with wrap-around here.
2173                         */
2174                        before = (u64 *)pos;
2175                        end = (u64 *)pos + flits;
2176                        txq = &q->q;
2177                        pos = (void *)inline_tx_skb_header(skb, &q->q,
2178                                                           (void *)pos,
2179                                                           hdr_len);
2180                        if (before > (u64 *)pos) {
2181                                left = (u8 *)end - (u8 *)txq->stat;
2182                                end = (void *)txq->desc + left;
2183                        }
2184
2185                        /* If current position is already at the end of the
2186                         * ofld queue, reset the current to point to
2187                         * start of the queue and update the end ptr as well.
2188                         */
2189                        if (pos == (u64 *)txq->stat) {
2190                                left = (u8 *)end - (u8 *)txq->stat;
2191                                end = (void *)txq->desc + left;
2192                                pos = (void *)txq->desc;
2193                        }
2194
2195                        cxgb4_write_sgl(skb, &q->q, (void *)pos,
2196                                        end, hdr_len,
2197                                        (dma_addr_t *)skb->head);
2198#ifdef CONFIG_NEED_DMA_MAP_STATE
2199                        skb->dev = q->adap->port[0];
2200                        skb->destructor = deferred_unmap_destructor;
2201#endif
2202                        last_desc = q->q.pidx + ndesc - 1;
2203                        if (last_desc >= q->q.size)
2204                                last_desc -= q->q.size;
2205                        q->q.sdesc[last_desc].skb = skb;
2206                }
2207
2208                txq_advance(&q->q, ndesc);
2209                written += ndesc;
2210                if (unlikely(written > 32)) {
2211                        cxgb4_ring_tx_db(q->adap, &q->q, written);
2212                        written = 0;
2213                }
2214
2215                /* Reacquire the Pending Send Queue Lock so we can unlink the
2216                 * skb we've just successfully transferred to the TX Ring and
2217                 * loop for the next skb which may be at the head of the
2218                 * Pending Send Queue.
2219                 */
2220                spin_lock(&q->sendq.lock);
2221                __skb_unlink(skb, &q->sendq);
2222                if (is_ofld_imm(skb))
2223                        kfree_skb(skb);
2224        }
2225        if (likely(written))
2226                cxgb4_ring_tx_db(q->adap, &q->q, written);
2227
2228        /*Indicate that no thread is processing the Pending Send Queue
2229         * currently.
2230         */
2231        q->service_ofldq_running = false;
2232}
2233
2234/**
2235 *      ofld_xmit - send a packet through an offload queue
2236 *      @q: the Tx offload queue
2237 *      @skb: the packet
2238 *
2239 *      Send an offload packet through an SGE offload queue.
2240 */
2241static int ofld_xmit(struct sge_uld_txq *q, struct sk_buff *skb)
2242{
2243        skb->priority = calc_tx_flits_ofld(skb);       /* save for restart */
2244        spin_lock(&q->sendq.lock);
2245
2246        /* Queue the new skb onto the Offload Queue's Pending Send Queue.  If
2247         * that results in this new skb being the only one on the queue, start
2248         * servicing it.  If there are other skbs already on the list, then
2249         * either the queue is currently being processed or it's been stopped
2250         * for some reason and it'll be restarted at a later time.  Restart
2251         * paths are triggered by events like experiencing a DMA Mapping Error
2252         * or filling the Hardware TX Ring.
2253         */
2254        __skb_queue_tail(&q->sendq, skb);
2255        if (q->sendq.qlen == 1)
2256                service_ofldq(q);
2257
2258        spin_unlock(&q->sendq.lock);
2259        return NET_XMIT_SUCCESS;
2260}
2261
2262/**
2263 *      restart_ofldq - restart a suspended offload queue
2264 *      @data: the offload queue to restart
2265 *
2266 *      Resumes transmission on a suspended Tx offload queue.
2267 */
2268static void restart_ofldq(unsigned long data)
2269{
2270        struct sge_uld_txq *q = (struct sge_uld_txq *)data;
2271
2272        spin_lock(&q->sendq.lock);
2273        q->full = 0;            /* the queue actually is completely empty now */
2274        service_ofldq(q);
2275        spin_unlock(&q->sendq.lock);
2276}
2277
2278/**
2279 *      skb_txq - return the Tx queue an offload packet should use
2280 *      @skb: the packet
2281 *
2282 *      Returns the Tx queue an offload packet should use as indicated by bits
2283 *      1-15 in the packet's queue_mapping.
2284 */
2285static inline unsigned int skb_txq(const struct sk_buff *skb)
2286{
2287        return skb->queue_mapping >> 1;
2288}
2289
2290/**
2291 *      is_ctrl_pkt - return whether an offload packet is a control packet
2292 *      @skb: the packet
2293 *
2294 *      Returns whether an offload packet should use an OFLD or a CTRL
2295 *      Tx queue as indicated by bit 0 in the packet's queue_mapping.
2296 */
2297static inline unsigned int is_ctrl_pkt(const struct sk_buff *skb)
2298{
2299        return skb->queue_mapping & 1;
2300}
2301
2302static inline int uld_send(struct adapter *adap, struct sk_buff *skb,
2303                           unsigned int tx_uld_type)
2304{
2305        struct sge_uld_txq_info *txq_info;
2306        struct sge_uld_txq *txq;
2307        unsigned int idx = skb_txq(skb);
2308
2309        if (unlikely(is_ctrl_pkt(skb))) {
2310                /* Single ctrl queue is a requirement for LE workaround path */
2311                if (adap->tids.nsftids)
2312                        idx = 0;
2313                return ctrl_xmit(&adap->sge.ctrlq[idx], skb);
2314        }
2315
2316        txq_info = adap->sge.uld_txq_info[tx_uld_type];
2317        if (unlikely(!txq_info)) {
2318                WARN_ON(true);
2319                return NET_XMIT_DROP;
2320        }
2321
2322        txq = &txq_info->uldtxq[idx];
2323        return ofld_xmit(txq, skb);
2324}
2325
2326/**
2327 *      t4_ofld_send - send an offload packet
2328 *      @adap: the adapter
2329 *      @skb: the packet
2330 *
2331 *      Sends an offload packet.  We use the packet queue_mapping to select the
2332 *      appropriate Tx queue as follows: bit 0 indicates whether the packet
2333 *      should be sent as regular or control, bits 1-15 select the queue.
2334 */
2335int t4_ofld_send(struct adapter *adap, struct sk_buff *skb)
2336{
2337        int ret;
2338
2339        local_bh_disable();
2340        ret = uld_send(adap, skb, CXGB4_TX_OFLD);
2341        local_bh_enable();
2342        return ret;
2343}
2344
2345/**
2346 *      cxgb4_ofld_send - send an offload packet
2347 *      @dev: the net device
2348 *      @skb: the packet
2349 *
2350 *      Sends an offload packet.  This is an exported version of @t4_ofld_send,
2351 *      intended for ULDs.
2352 */
2353int cxgb4_ofld_send(struct net_device *dev, struct sk_buff *skb)
2354{
2355        return t4_ofld_send(netdev2adap(dev), skb);
2356}
2357EXPORT_SYMBOL(cxgb4_ofld_send);
2358
2359static void *inline_tx_header(const void *src,
2360                              const struct sge_txq *q,
2361                              void *pos, int length)
2362{
2363        int left = (void *)q->stat - pos;
2364        u64 *p;
2365
2366        if (likely(length <= left)) {
2367                memcpy(pos, src, length);
2368                pos += length;
2369        } else {
2370                memcpy(pos, src, left);
2371                memcpy(q->desc, src + left, length - left);
2372                pos = (void *)q->desc + (length - left);
2373        }
2374        /* 0-pad to multiple of 16 */
2375        p = PTR_ALIGN(pos, 8);
2376        if ((uintptr_t)p & 8) {
2377                *p = 0;
2378                return p + 1;
2379        }
2380        return p;
2381}
2382
2383/**
2384 *      ofld_xmit_direct - copy a WR into offload queue
2385 *      @q: the Tx offload queue
2386 *      @src: location of WR
2387 *      @len: WR length
2388 *
2389 *      Copy an immediate WR into an uncontended SGE offload queue.
2390 */
2391static int ofld_xmit_direct(struct sge_uld_txq *q, const void *src,
2392                            unsigned int len)
2393{
2394        unsigned int ndesc;
2395        int credits;
2396        u64 *pos;
2397
2398        /* Use the lower limit as the cut-off */
2399        if (len > MAX_IMM_OFLD_TX_DATA_WR_LEN) {
2400                WARN_ON(1);
2401                return NET_XMIT_DROP;
2402        }
2403
2404        /* Don't return NET_XMIT_CN here as the current
2405         * implementation doesn't queue the request
2406         * using an skb when the following conditions not met
2407         */
2408        if (!spin_trylock(&q->sendq.lock))
2409                return NET_XMIT_DROP;
2410
2411        if (q->full || !skb_queue_empty(&q->sendq) ||
2412            q->service_ofldq_running) {
2413                spin_unlock(&q->sendq.lock);
2414                return NET_XMIT_DROP;
2415        }
2416        ndesc = flits_to_desc(DIV_ROUND_UP(len, 8));
2417        credits = txq_avail(&q->q) - ndesc;
2418        pos = (u64 *)&q->q.desc[q->q.pidx];
2419
2420        /* ofldtxq_stop modifies WR header in-situ */
2421        inline_tx_header(src, &q->q, pos, len);
2422        if (unlikely(credits < TXQ_STOP_THRES))
2423                ofldtxq_stop(q, (struct fw_wr_hdr *)pos);
2424        txq_advance(&q->q, ndesc);
2425        cxgb4_ring_tx_db(q->adap, &q->q, ndesc);
2426
2427        spin_unlock(&q->sendq.lock);
2428        return NET_XMIT_SUCCESS;
2429}
2430
2431int cxgb4_immdata_send(struct net_device *dev, unsigned int idx,
2432                       const void *src, unsigned int len)
2433{
2434        struct sge_uld_txq_info *txq_info;
2435        struct sge_uld_txq *txq;
2436        struct adapter *adap;
2437        int ret;
2438
2439        adap = netdev2adap(dev);
2440
2441        local_bh_disable();
2442        txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD];
2443        if (unlikely(!txq_info)) {
2444                WARN_ON(true);
2445                local_bh_enable();
2446                return NET_XMIT_DROP;
2447        }
2448        txq = &txq_info->uldtxq[idx];
2449
2450        ret = ofld_xmit_direct(txq, src, len);
2451        local_bh_enable();
2452        return net_xmit_eval(ret);
2453}
2454EXPORT_SYMBOL(cxgb4_immdata_send);
2455
2456/**
2457 *      t4_crypto_send - send crypto packet
2458 *      @adap: the adapter
2459 *      @skb: the packet
2460 *
2461 *      Sends crypto packet.  We use the packet queue_mapping to select the
2462 *      appropriate Tx queue as follows: bit 0 indicates whether the packet
2463 *      should be sent as regular or control, bits 1-15 select the queue.
2464 */
2465static int t4_crypto_send(struct adapter *adap, struct sk_buff *skb)
2466{
2467        int ret;
2468
2469        local_bh_disable();
2470        ret = uld_send(adap, skb, CXGB4_TX_CRYPTO);
2471        local_bh_enable();
2472        return ret;
2473}
2474
2475/**
2476 *      cxgb4_crypto_send - send crypto packet
2477 *      @dev: the net device
2478 *      @skb: the packet
2479 *
2480 *      Sends crypto packet.  This is an exported version of @t4_crypto_send,
2481 *      intended for ULDs.
2482 */
2483int cxgb4_crypto_send(struct net_device *dev, struct sk_buff *skb)
2484{
2485        return t4_crypto_send(netdev2adap(dev), skb);
2486}
2487EXPORT_SYMBOL(cxgb4_crypto_send);
2488
2489static inline void copy_frags(struct sk_buff *skb,
2490                              const struct pkt_gl *gl, unsigned int offset)
2491{
2492        int i;
2493
2494        /* usually there's just one frag */
2495        __skb_fill_page_desc(skb, 0, gl->frags[0].page,
2496                             gl->frags[0].offset + offset,
2497                             gl->frags[0].size - offset);
2498        skb_shinfo(skb)->nr_frags = gl->nfrags;
2499        for (i = 1; i < gl->nfrags; i++)
2500                __skb_fill_page_desc(skb, i, gl->frags[i].page,
2501                                     gl->frags[i].offset,
2502                                     gl->frags[i].size);
2503
2504        /* get a reference to the last page, we don't own it */
2505        get_page(gl->frags[gl->nfrags - 1].page);
2506}
2507
2508/**
2509 *      cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list
2510 *      @gl: the gather list
2511 *      @skb_len: size of sk_buff main body if it carries fragments
2512 *      @pull_len: amount of data to move to the sk_buff's main body
2513 *
2514 *      Builds an sk_buff from the given packet gather list.  Returns the
2515 *      sk_buff or %NULL if sk_buff allocation failed.
2516 */
2517struct sk_buff *cxgb4_pktgl_to_skb(const struct pkt_gl *gl,
2518                                   unsigned int skb_len, unsigned int pull_len)
2519{
2520        struct sk_buff *skb;
2521
2522        /*
2523         * Below we rely on RX_COPY_THRES being less than the smallest Rx buffer
2524         * size, which is expected since buffers are at least PAGE_SIZEd.
2525         * In this case packets up to RX_COPY_THRES have only one fragment.
2526         */
2527        if (gl->tot_len <= RX_COPY_THRES) {
2528                skb = dev_alloc_skb(gl->tot_len);
2529                if (unlikely(!skb))
2530                        goto out;
2531                __skb_put(skb, gl->tot_len);
2532                skb_copy_to_linear_data(skb, gl->va, gl->tot_len);
2533        } else {
2534                skb = dev_alloc_skb(skb_len);
2535                if (unlikely(!skb))
2536                        goto out;
2537                __skb_put(skb, pull_len);
2538                skb_copy_to_linear_data(skb, gl->va, pull_len);
2539
2540                copy_frags(skb, gl, pull_len);
2541                skb->len = gl->tot_len;
2542                skb->data_len = skb->len - pull_len;
2543                skb->truesize += skb->data_len;
2544        }
2545out:    return skb;
2546}
2547EXPORT_SYMBOL(cxgb4_pktgl_to_skb);
2548
2549/**
2550 *      t4_pktgl_free - free a packet gather list
2551 *      @gl: the gather list
2552 *
2553 *      Releases the pages of a packet gather list.  We do not own the last
2554 *      page on the list and do not free it.
2555 */
2556static void t4_pktgl_free(const struct pkt_gl *gl)
2557{
2558        int n;
2559        const struct page_frag *p;
2560
2561        for (p = gl->frags, n = gl->nfrags - 1; n--; p++)
2562                put_page(p->page);
2563}
2564
2565/*
2566 * Process an MPS trace packet.  Give it an unused protocol number so it won't
2567 * be delivered to anyone and send it to the stack for capture.
2568 */
2569static noinline int handle_trace_pkt(struct adapter *adap,
2570                                     const struct pkt_gl *gl)
2571{
2572        struct sk_buff *skb;
2573
2574        skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
2575        if (unlikely(!skb)) {
2576                t4_pktgl_free(gl);
2577                return 0;
2578        }
2579
2580        if (is_t4(adap->params.chip))
2581                __skb_pull(skb, sizeof(struct cpl_trace_pkt));
2582        else
2583                __skb_pull(skb, sizeof(struct cpl_t5_trace_pkt));
2584
2585        skb_reset_mac_header(skb);
2586        skb->protocol = htons(0xffff);
2587        skb->dev = adap->port[0];
2588        netif_receive_skb(skb);
2589        return 0;
2590}
2591
2592/**
2593 * cxgb4_sgetim_to_hwtstamp - convert sge time stamp to hw time stamp
2594 * @adap: the adapter
2595 * @hwtstamps: time stamp structure to update
2596 * @sgetstamp: 60bit iqe timestamp
2597 *
2598 * Every ingress queue entry has the 60-bit timestamp, convert that timestamp
2599 * which is in Core Clock ticks into ktime_t and assign it
2600 **/
2601static void cxgb4_sgetim_to_hwtstamp(struct adapter *adap,
2602                                     struct skb_shared_hwtstamps *hwtstamps,
2603                                     u64 sgetstamp)
2604{
2605        u64 ns;
2606        u64 tmp = (sgetstamp * 1000 * 1000 + adap->params.vpd.cclk / 2);
2607
2608        ns = div_u64(tmp, adap->params.vpd.cclk);
2609
2610        memset(hwtstamps, 0, sizeof(*hwtstamps));
2611        hwtstamps->hwtstamp = ns_to_ktime(ns);
2612}
2613
2614static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
2615                   const struct cpl_rx_pkt *pkt, unsigned long tnl_hdr_len)
2616{
2617        struct adapter *adapter = rxq->rspq.adap;
2618        struct sge *s = &adapter->sge;
2619        struct port_info *pi;
2620        int ret;
2621        struct sk_buff *skb;
2622
2623        skb = napi_get_frags(&rxq->rspq.napi);
2624        if (unlikely(!skb)) {
2625                t4_pktgl_free(gl);
2626                rxq->stats.rx_drops++;
2627                return;
2628        }
2629
2630        copy_frags(skb, gl, s->pktshift);
2631        if (tnl_hdr_len)
2632                skb->csum_level = 1;
2633        skb->len = gl->tot_len - s->pktshift;
2634        skb->data_len = skb->len;
2635        skb->truesize += skb->data_len;
2636        skb->ip_summed = CHECKSUM_UNNECESSARY;
2637        skb_record_rx_queue(skb, rxq->rspq.idx);
2638        skb_mark_napi_id(skb, &rxq->rspq.napi);
2639        pi = netdev_priv(skb->dev);
2640        if (pi->rxtstamp)
2641                cxgb4_sgetim_to_hwtstamp(adapter, skb_hwtstamps(skb),
2642                                         gl->sgetstamp);
2643        if (rxq->rspq.netdev->features & NETIF_F_RXHASH)
2644                skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
2645                             PKT_HASH_TYPE_L3);
2646
2647        if (unlikely(pkt->vlan_ex)) {
2648                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
2649                rxq->stats.vlan_ex++;
2650        }
2651        ret = napi_gro_frags(&rxq->rspq.napi);
2652        if (ret == GRO_HELD)
2653                rxq->stats.lro_pkts++;
2654        else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE)
2655                rxq->stats.lro_merged++;
2656        rxq->stats.pkts++;
2657        rxq->stats.rx_cso++;
2658}
2659
2660enum {
2661        RX_NON_PTP_PKT = 0,
2662        RX_PTP_PKT_SUC = 1,
2663        RX_PTP_PKT_ERR = 2
2664};
2665
2666/**
2667 *     t4_systim_to_hwstamp - read hardware time stamp
2668 *     @adap: the adapter
2669 *     @skb: the packet
2670 *
2671 *     Read Time Stamp from MPS packet and insert in skb which
2672 *     is forwarded to PTP application
2673 */
2674static noinline int t4_systim_to_hwstamp(struct adapter *adapter,
2675                                         struct sk_buff *skb)
2676{
2677        struct skb_shared_hwtstamps *hwtstamps;
2678        struct cpl_rx_mps_pkt *cpl = NULL;
2679        unsigned char *data;
2680        int offset;
2681
2682        cpl = (struct cpl_rx_mps_pkt *)skb->data;
2683        if (!(CPL_RX_MPS_PKT_TYPE_G(ntohl(cpl->op_to_r1_hi)) &
2684             X_CPL_RX_MPS_PKT_TYPE_PTP))
2685                return RX_PTP_PKT_ERR;
2686
2687        data = skb->data + sizeof(*cpl);
2688        skb_pull(skb, 2 * sizeof(u64) + sizeof(struct cpl_rx_mps_pkt));
2689        offset = ETH_HLEN + IPV4_HLEN(skb->data) + UDP_HLEN;
2690        if (skb->len < offset + OFF_PTP_SEQUENCE_ID + sizeof(short))
2691                return RX_PTP_PKT_ERR;
2692
2693        hwtstamps = skb_hwtstamps(skb);
2694        memset(hwtstamps, 0, sizeof(*hwtstamps));
2695        hwtstamps->hwtstamp = ns_to_ktime(be64_to_cpu(*((u64 *)data)));
2696
2697        return RX_PTP_PKT_SUC;
2698}
2699
2700/**
2701 *     t4_rx_hststamp - Recv PTP Event Message
2702 *     @adap: the adapter
2703 *     @rsp: the response queue descriptor holding the RX_PKT message
2704 *     @skb: the packet
2705 *
2706 *     PTP enabled and MPS packet, read HW timestamp
2707 */
2708static int t4_rx_hststamp(struct adapter *adapter, const __be64 *rsp,
2709                          struct sge_eth_rxq *rxq, struct sk_buff *skb)
2710{
2711        int ret;
2712
2713        if (unlikely((*(u8 *)rsp == CPL_RX_MPS_PKT) &&
2714                     !is_t4(adapter->params.chip))) {
2715                ret = t4_systim_to_hwstamp(adapter, skb);
2716                if (ret == RX_PTP_PKT_ERR) {
2717                        kfree_skb(skb);
2718                        rxq->stats.rx_drops++;
2719                }
2720                return ret;
2721        }
2722        return RX_NON_PTP_PKT;
2723}
2724
2725/**
2726 *      t4_tx_hststamp - Loopback PTP Transmit Event Message
2727 *      @adap: the adapter
2728 *      @skb: the packet
2729 *      @dev: the ingress net device
2730 *
2731 *      Read hardware timestamp for the loopback PTP Tx event message
2732 */
2733static int t4_tx_hststamp(struct adapter *adapter, struct sk_buff *skb,
2734                          struct net_device *dev)
2735{
2736        struct port_info *pi = netdev_priv(dev);
2737
2738        if (!is_t4(adapter->params.chip) && adapter->ptp_tx_skb) {
2739                cxgb4_ptp_read_hwstamp(adapter, pi);
2740                kfree_skb(skb);
2741                return 0;
2742        }
2743        return 1;
2744}
2745
2746/**
2747 *      t4_ethrx_handler - process an ingress ethernet packet
2748 *      @q: the response queue that received the packet
2749 *      @rsp: the response queue descriptor holding the RX_PKT message
2750 *      @si: the gather list of packet fragments
2751 *
2752 *      Process an ingress ethernet packet and deliver it to the stack.
2753 */
2754int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
2755                     const struct pkt_gl *si)
2756{
2757        bool csum_ok;
2758        struct sk_buff *skb;
2759        const struct cpl_rx_pkt *pkt;
2760        struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
2761        struct adapter *adapter = q->adap;
2762        struct sge *s = &q->adap->sge;
2763        int cpl_trace_pkt = is_t4(q->adap->params.chip) ?
2764                            CPL_TRACE_PKT : CPL_TRACE_PKT_T5;
2765        u16 err_vec, tnl_hdr_len = 0;
2766        struct port_info *pi;
2767        int ret = 0;
2768
2769        if (unlikely(*(u8 *)rsp == cpl_trace_pkt))
2770                return handle_trace_pkt(q->adap, si);
2771
2772        pkt = (const struct cpl_rx_pkt *)rsp;
2773        /* Compressed error vector is enabled for T6 only */
2774        if (q->adap->params.tp.rx_pkt_encap) {
2775                err_vec = T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt->err_vec));
2776                tnl_hdr_len = T6_RX_TNLHDR_LEN_G(ntohs(pkt->err_vec));
2777        } else {
2778                err_vec = be16_to_cpu(pkt->err_vec);
2779        }
2780
2781        csum_ok = pkt->csum_calc && !err_vec &&
2782                  (q->netdev->features & NETIF_F_RXCSUM);
2783
2784        if (err_vec)
2785                rxq->stats.bad_rx_pkts++;
2786
2787        if (((pkt->l2info & htonl(RXF_TCP_F)) ||
2788             tnl_hdr_len) &&
2789            (q->netdev->features & NETIF_F_GRO) && csum_ok && !pkt->ip_frag) {
2790                do_gro(rxq, si, pkt, tnl_hdr_len);
2791                return 0;
2792        }
2793
2794        skb = cxgb4_pktgl_to_skb(si, RX_PKT_SKB_LEN, RX_PULL_LEN);
2795        if (unlikely(!skb)) {
2796                t4_pktgl_free(si);
2797                rxq->stats.rx_drops++;
2798                return 0;
2799        }
2800        pi = netdev_priv(q->netdev);
2801
2802        /* Handle PTP Event Rx packet */
2803        if (unlikely(pi->ptp_enable)) {
2804                ret = t4_rx_hststamp(adapter, rsp, rxq, skb);
2805                if (ret == RX_PTP_PKT_ERR)
2806                        return 0;
2807        }
2808        if (likely(!ret))
2809                __skb_pull(skb, s->pktshift); /* remove ethernet header pad */
2810
2811        /* Handle the PTP Event Tx Loopback packet */
2812        if (unlikely(pi->ptp_enable && !ret &&
2813                     (pkt->l2info & htonl(RXF_UDP_F)) &&
2814                     cxgb4_ptp_is_ptp_rx(skb))) {
2815                if (!t4_tx_hststamp(adapter, skb, q->netdev))
2816                        return 0;
2817        }
2818
2819        skb->protocol = eth_type_trans(skb, q->netdev);
2820        skb_record_rx_queue(skb, q->idx);
2821        if (skb->dev->features & NETIF_F_RXHASH)
2822                skb_set_hash(skb, (__force u32)pkt->rsshdr.hash_val,
2823                             PKT_HASH_TYPE_L3);
2824
2825        rxq->stats.pkts++;
2826
2827        if (pi->rxtstamp)
2828                cxgb4_sgetim_to_hwtstamp(q->adap, skb_hwtstamps(skb),
2829                                         si->sgetstamp);
2830        if (csum_ok && (pkt->l2info & htonl(RXF_UDP_F | RXF_TCP_F))) {
2831                if (!pkt->ip_frag) {
2832                        skb->ip_summed = CHECKSUM_UNNECESSARY;
2833                        rxq->stats.rx_cso++;
2834                } else if (pkt->l2info & htonl(RXF_IP_F)) {
2835                        __sum16 c = (__force __sum16)pkt->csum;
2836                        skb->csum = csum_unfold(c);
2837
2838                        if (tnl_hdr_len) {
2839                                skb->ip_summed = CHECKSUM_UNNECESSARY;
2840                                skb->csum_level = 1;
2841                        } else {
2842                                skb->ip_summed = CHECKSUM_COMPLETE;
2843                        }
2844                        rxq->stats.rx_cso++;
2845                }
2846        } else
2847                skb_checksum_none_assert(skb);
2848
2849        if (unlikely(pkt->vlan_ex)) {
2850                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(pkt->vlan));
2851                rxq->stats.vlan_ex++;
2852        }
2853        skb_mark_napi_id(skb, &q->napi);
2854        netif_receive_skb(skb);
2855        return 0;
2856}
2857
2858/**
2859 *      restore_rx_bufs - put back a packet's Rx buffers
2860 *      @si: the packet gather list
2861 *      @q: the SGE free list
2862 *      @frags: number of FL buffers to restore
2863 *
2864 *      Puts back on an FL the Rx buffers associated with @si.  The buffers
2865 *      have already been unmapped and are left unmapped, we mark them so to
2866 *      prevent further unmapping attempts.
2867 *
2868 *      This function undoes a series of @unmap_rx_buf calls when we find out
2869 *      that the current packet can't be processed right away afterall and we
2870 *      need to come back to it later.  This is a very rare event and there's
2871 *      no effort to make this particularly efficient.
2872 */
2873static void restore_rx_bufs(const struct pkt_gl *si, struct sge_fl *q,
2874                            int frags)
2875{
2876        struct rx_sw_desc *d;
2877
2878        while (frags--) {
2879                if (q->cidx == 0)
2880                        q->cidx = q->size - 1;
2881                else
2882                        q->cidx--;
2883                d = &q->sdesc[q->cidx];
2884                d->page = si->frags[frags].page;
2885                d->dma_addr |= RX_UNMAPPED_BUF;
2886                q->avail++;
2887        }
2888}
2889
2890/**
2891 *      is_new_response - check if a response is newly written
2892 *      @r: the response descriptor
2893 *      @q: the response queue
2894 *
2895 *      Returns true if a response descriptor contains a yet unprocessed
2896 *      response.
2897 */
2898static inline bool is_new_response(const struct rsp_ctrl *r,
2899                                   const struct sge_rspq *q)
2900{
2901        return (r->type_gen >> RSPD_GEN_S) == q->gen;
2902}
2903
2904/**
2905 *      rspq_next - advance to the next entry in a response queue
2906 *      @q: the queue
2907 *
2908 *      Updates the state of a response queue to advance it to the next entry.
2909 */
2910static inline void rspq_next(struct sge_rspq *q)
2911{
2912        q->cur_desc = (void *)q->cur_desc + q->iqe_len;
2913        if (unlikely(++q->cidx == q->size)) {
2914                q->cidx = 0;
2915                q->gen ^= 1;
2916                q->cur_desc = q->desc;
2917        }
2918}
2919
2920/**
2921 *      process_responses - process responses from an SGE response queue
2922 *      @q: the ingress queue to process
2923 *      @budget: how many responses can be processed in this round
2924 *
2925 *      Process responses from an SGE response queue up to the supplied budget.
2926 *      Responses include received packets as well as control messages from FW
2927 *      or HW.
2928 *
2929 *      Additionally choose the interrupt holdoff time for the next interrupt
2930 *      on this queue.  If the system is under memory shortage use a fairly
2931 *      long delay to help recovery.
2932 */
2933static int process_responses(struct sge_rspq *q, int budget)
2934{
2935        int ret, rsp_type;
2936        int budget_left = budget;
2937        const struct rsp_ctrl *rc;
2938        struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
2939        struct adapter *adapter = q->adap;
2940        struct sge *s = &adapter->sge;
2941
2942        while (likely(budget_left)) {
2943                rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
2944                if (!is_new_response(rc, q)) {
2945                        if (q->flush_handler)
2946                                q->flush_handler(q);
2947                        break;
2948                }
2949
2950                dma_rmb();
2951                rsp_type = RSPD_TYPE_G(rc->type_gen);
2952                if (likely(rsp_type == RSPD_TYPE_FLBUF_X)) {
2953                        struct page_frag *fp;
2954                        struct pkt_gl si;
2955                        const struct rx_sw_desc *rsd;
2956                        u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags;
2957
2958                        if (len & RSPD_NEWBUF_F) {
2959                                if (likely(q->offset > 0)) {
2960                                        free_rx_bufs(q->adap, &rxq->fl, 1);
2961                                        q->offset = 0;
2962                                }
2963                                len = RSPD_LEN_G(len);
2964                        }
2965                        si.tot_len = len;
2966
2967                        /* gather packet fragments */
2968                        for (frags = 0, fp = si.frags; ; frags++, fp++) {
2969                                rsd = &rxq->fl.sdesc[rxq->fl.cidx];
2970                                bufsz = get_buf_size(adapter, rsd);
2971                                fp->page = rsd->page;
2972                                fp->offset = q->offset;
2973                                fp->size = min(bufsz, len);
2974                                len -= fp->size;
2975                                if (!len)
2976                                        break;
2977                                unmap_rx_buf(q->adap, &rxq->fl);
2978                        }
2979
2980                        si.sgetstamp = SGE_TIMESTAMP_G(
2981                                        be64_to_cpu(rc->last_flit));
2982                        /*
2983                         * Last buffer remains mapped so explicitly make it
2984                         * coherent for CPU access.
2985                         */
2986                        dma_sync_single_for_cpu(q->adap->pdev_dev,
2987                                                get_buf_addr(rsd),
2988                                                fp->size, DMA_FROM_DEVICE);
2989
2990                        si.va = page_address(si.frags[0].page) +
2991                                si.frags[0].offset;
2992                        prefetch(si.va);
2993
2994                        si.nfrags = frags + 1;
2995                        ret = q->handler(q, q->cur_desc, &si);
2996                        if (likely(ret == 0))
2997                                q->offset += ALIGN(fp->size, s->fl_align);
2998                        else
2999                                restore_rx_bufs(&si, &rxq->fl, frags);
3000                } else if (likely(rsp_type == RSPD_TYPE_CPL_X)) {
3001                        ret = q->handler(q, q->cur_desc, NULL);
3002                } else {
3003                        ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN);
3004                }
3005
3006                if (unlikely(ret)) {
3007                        /* couldn't process descriptor, back off for recovery */
3008                        q->next_intr_params = QINTR_TIMER_IDX_V(NOMEM_TMR_IDX);
3009                        break;
3010                }
3011
3012                rspq_next(q);
3013                budget_left--;
3014        }
3015
3016        if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 16)
3017                __refill_fl(q->adap, &rxq->fl);
3018        return budget - budget_left;
3019}
3020
3021/**
3022 *      napi_rx_handler - the NAPI handler for Rx processing
3023 *      @napi: the napi instance
3024 *      @budget: how many packets we can process in this round
3025 *
3026 *      Handler for new data events when using NAPI.  This does not need any
3027 *      locking or protection from interrupts as data interrupts are off at
3028 *      this point and other adapter interrupts do not interfere (the latter
3029 *      in not a concern at all with MSI-X as non-data interrupts then have
3030 *      a separate handler).
3031 */
3032static int napi_rx_handler(struct napi_struct *napi, int budget)
3033{
3034        unsigned int params;
3035        struct sge_rspq *q = container_of(napi, struct sge_rspq, napi);
3036        int work_done;
3037        u32 val;
3038
3039        work_done = process_responses(q, budget);
3040        if (likely(work_done < budget)) {
3041                int timer_index;
3042
3043                napi_complete_done(napi, work_done);
3044                timer_index = QINTR_TIMER_IDX_G(q->next_intr_params);
3045
3046                if (q->adaptive_rx) {
3047                        if (work_done > max(timer_pkt_quota[timer_index],
3048                                            MIN_NAPI_WORK))
3049                                timer_index = (timer_index + 1);
3050                        else
3051                                timer_index = timer_index - 1;
3052
3053                        timer_index = clamp(timer_index, 0, SGE_TIMERREGS - 1);
3054                        q->next_intr_params =
3055                                        QINTR_TIMER_IDX_V(timer_index) |
3056                                        QINTR_CNT_EN_V(0);
3057                        params = q->next_intr_params;
3058                } else {
3059                        params = q->next_intr_params;
3060                        q->next_intr_params = q->intr_params;
3061                }
3062        } else
3063                params = QINTR_TIMER_IDX_V(7);
3064
3065        val = CIDXINC_V(work_done) | SEINTARM_V(params);
3066
3067        /* If we don't have access to the new User GTS (T5+), use the old
3068         * doorbell mechanism; otherwise use the new BAR2 mechanism.
3069         */
3070        if (unlikely(q->bar2_addr == NULL)) {
3071                t4_write_reg(q->adap, MYPF_REG(SGE_PF_GTS_A),
3072                             val | INGRESSQID_V((u32)q->cntxt_id));
3073        } else {
3074                writel(val | INGRESSQID_V(q->bar2_qid),
3075                       q->bar2_addr + SGE_UDB_GTS);
3076                wmb();
3077        }
3078        return work_done;
3079}
3080
3081/*
3082 * The MSI-X interrupt handler for an SGE response queue.
3083 */
3084irqreturn_t t4_sge_intr_msix(int irq, void *cookie)
3085{
3086        struct sge_rspq *q = cookie;
3087
3088        napi_schedule(&q->napi);
3089        return IRQ_HANDLED;
3090}
3091
3092/*
3093 * Process the indirect interrupt entries in the interrupt queue and kick off
3094 * NAPI for each queue that has generated an entry.
3095 */
3096static unsigned int process_intrq(struct adapter *adap)
3097{
3098        unsigned int credits;
3099        const struct rsp_ctrl *rc;
3100        struct sge_rspq *q = &adap->sge.intrq;
3101        u32 val;
3102
3103        spin_lock(&adap->sge.intrq_lock);
3104        for (credits = 0; ; credits++) {
3105                rc = (void *)q->cur_desc + (q->iqe_len - sizeof(*rc));
3106                if (!is_new_response(rc, q))
3107                        break;
3108
3109                dma_rmb();
3110                if (RSPD_TYPE_G(rc->type_gen) == RSPD_TYPE_INTR_X) {
3111                        unsigned int qid = ntohl(rc->pldbuflen_qid);
3112
3113                        qid -= adap->sge.ingr_start;
3114                        napi_schedule(&adap->sge.ingr_map[qid]->napi);
3115                }
3116
3117                rspq_next(q);
3118        }
3119
3120        val =  CIDXINC_V(credits) | SEINTARM_V(q->intr_params);
3121
3122        /* If we don't have access to the new User GTS (T5+), use the old
3123         * doorbell mechanism; otherwise use the new BAR2 mechanism.
3124         */
3125        if (unlikely(q->bar2_addr == NULL)) {
3126                t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
3127                             val | INGRESSQID_V(q->cntxt_id));
3128        } else {
3129                writel(val | INGRESSQID_V(q->bar2_qid),
3130                       q->bar2_addr + SGE_UDB_GTS);
3131                wmb();
3132        }
3133        spin_unlock(&adap->sge.intrq_lock);
3134        return credits;
3135}
3136
3137/*
3138 * The MSI interrupt handler, which handles data events from SGE response queues
3139 * as well as error and other async events as they all use the same MSI vector.
3140 */
3141static irqreturn_t t4_intr_msi(int irq, void *cookie)
3142{
3143        struct adapter *adap = cookie;
3144
3145        if (adap->flags & MASTER_PF)
3146                t4_slow_intr_handler(adap);
3147        process_intrq(adap);
3148        return IRQ_HANDLED;
3149}
3150
3151/*
3152 * Interrupt handler for legacy INTx interrupts.
3153 * Handles data events from SGE response queues as well as error and other
3154 * async events as they all use the same interrupt line.
3155 */
3156static irqreturn_t t4_intr_intx(int irq, void *cookie)
3157{
3158        struct adapter *adap = cookie;
3159
3160        t4_write_reg(adap, MYPF_REG(PCIE_PF_CLI_A), 0);
3161        if (((adap->flags & MASTER_PF) && t4_slow_intr_handler(adap)) |
3162            process_intrq(adap))
3163                return IRQ_HANDLED;
3164        return IRQ_NONE;             /* probably shared interrupt */
3165}
3166
3167/**
3168 *      t4_intr_handler - select the top-level interrupt handler
3169 *      @adap: the adapter
3170 *
3171 *      Selects the top-level interrupt handler based on the type of interrupts
3172 *      (MSI-X, MSI, or INTx).
3173 */
3174irq_handler_t t4_intr_handler(struct adapter *adap)
3175{
3176        if (adap->flags & USING_MSIX)
3177                return t4_sge_intr_msix;
3178        if (adap->flags & USING_MSI)
3179                return t4_intr_msi;
3180        return t4_intr_intx;
3181}
3182
3183static void sge_rx_timer_cb(struct timer_list *t)
3184{
3185        unsigned long m;
3186        unsigned int i;
3187        struct adapter *adap = from_timer(adap, t, sge.rx_timer);
3188        struct sge *s = &adap->sge;
3189
3190        for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
3191                for (m = s->starving_fl[i]; m; m &= m - 1) {
3192                        struct sge_eth_rxq *rxq;
3193                        unsigned int id = __ffs(m) + i * BITS_PER_LONG;
3194                        struct sge_fl *fl = s->egr_map[id];
3195
3196                        clear_bit(id, s->starving_fl);
3197                        smp_mb__after_clear_bit();
3198
3199                        if (fl_starving(adap, fl)) {
3200                                rxq = container_of(fl, struct sge_eth_rxq, fl);
3201                                if (napi_reschedule(&rxq->rspq.napi))
3202                                        fl->starving++;
3203                                else
3204                                        set_bit(id, s->starving_fl);
3205                        }
3206                }
3207        /* The remainder of the SGE RX Timer Callback routine is dedicated to
3208         * global Master PF activities like checking for chip ingress stalls,
3209         * etc.
3210         */
3211        if (!(adap->flags & MASTER_PF))
3212                goto done;
3213
3214        t4_idma_monitor(adap, &s->idma_monitor, HZ, RX_QCHECK_PERIOD);
3215
3216done:
3217        mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD);
3218}
3219
3220static void sge_tx_timer_cb(struct timer_list *t)
3221{
3222        unsigned long m;
3223        unsigned int i, budget;
3224        struct adapter *adap = from_timer(adap, t, sge.tx_timer);
3225        struct sge *s = &adap->sge;
3226
3227        for (i = 0; i < BITS_TO_LONGS(s->egr_sz); i++)
3228                for (m = s->txq_maperr[i]; m; m &= m - 1) {
3229                        unsigned long id = __ffs(m) + i * BITS_PER_LONG;
3230                        struct sge_uld_txq *txq = s->egr_map[id];
3231
3232                        clear_bit(id, s->txq_maperr);
3233                        tasklet_schedule(&txq->qresume_tsk);
3234                }
3235
3236        if (!is_t4(adap->params.chip)) {
3237                struct sge_eth_txq *q = &s->ptptxq;
3238                int avail;
3239
3240                spin_lock(&adap->ptp_lock);
3241                avail = reclaimable(&q->q);
3242
3243                if (avail) {
3244                        free_tx_desc(adap, &q->q, avail, false);
3245                        q->q.in_use -= avail;
3246                }
3247                spin_unlock(&adap->ptp_lock);
3248        }
3249
3250        budget = MAX_TIMER_TX_RECLAIM;
3251        i = s->ethtxq_rover;
3252        do {
3253                struct sge_eth_txq *q = &s->ethtxq[i];
3254
3255                if (q->q.in_use &&
3256                    time_after_eq(jiffies, q->txq->trans_start + HZ / 100) &&
3257                    __netif_tx_trylock(q->txq)) {
3258                        int avail = reclaimable(&q->q);
3259
3260                        if (avail) {
3261                                if (avail > budget)
3262                                        avail = budget;
3263
3264                                free_tx_desc(adap, &q->q, avail, true);
3265                                q->q.in_use -= avail;
3266                                budget -= avail;
3267                        }
3268                        __netif_tx_unlock(q->txq);
3269                }
3270
3271                if (++i >= s->ethqsets)
3272                        i = 0;
3273        } while (budget && i != s->ethtxq_rover);
3274        s->ethtxq_rover = i;
3275        mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2));
3276}
3277
3278/**
3279 *      bar2_address - return the BAR2 address for an SGE Queue's Registers
3280 *      @adapter: the adapter
3281 *      @qid: the SGE Queue ID
3282 *      @qtype: the SGE Queue Type (Egress or Ingress)
3283 *      @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
3284 *
3285 *      Returns the BAR2 address for the SGE Queue Registers associated with
3286 *      @qid. If BAR2 SGE Registers aren't available, returns NULL. Also
3287 *      returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
3288 *      Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID"
3289 *      Registers are supported (e.g. the Write Combining Doorbell Buffer).
3290 */
3291static void __iomem *bar2_address(struct adapter *adapter,
3292                                 unsigned int qid,
3293                                 enum t4_bar2_qtype qtype,
3294                                 unsigned int *pbar2_qid)
3295{
3296        u64 bar2_qoffset;
3297        int ret;
3298
3299        ret = t4_bar2_sge_qregs(adapter, qid, qtype, 0,
3300                                &bar2_qoffset, pbar2_qid);
3301        if (ret)
3302                return NULL;
3303        return adapter->bar2 + bar2_qoffset;
3304}
3305
3306/* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0
3307 * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map
3308 */
3309int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
3310                     struct net_device *dev, int intr_idx,
3311                     struct sge_fl *fl, rspq_handler_t hnd,
3312                     rspq_flush_handler_t flush_hnd, int cong)
3313{
3314        int ret, flsz = 0;
3315        struct fw_iq_cmd c;
3316        struct sge *s = &adap->sge;
3317        struct port_info *pi = netdev_priv(dev);
3318        int relaxed = !(adap->flags & ROOT_NO_RELAXED_ORDERING);
3319
3320        /* Size needs to be multiple of 16, including status entry. */
3321        iq->size = roundup(iq->size, 16);
3322
3323        iq->desc = alloc_ring(adap->pdev_dev, iq->size, iq->iqe_len, 0,
3324                              &iq->phys_addr, NULL, 0,
3325                              dev_to_node(adap->pdev_dev));
3326        if (!iq->desc)
3327                return -ENOMEM;
3328
3329        memset(&c, 0, sizeof(c));
3330        c.op_to_vfn = htonl(FW_CMD_OP_V(FW_IQ_CMD) | FW_CMD_REQUEST_F |
3331                            FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3332                            FW_IQ_CMD_PFN_V(adap->pf) | FW_IQ_CMD_VFN_V(0));
3333        c.alloc_to_len16 = htonl(FW_IQ_CMD_ALLOC_F | FW_IQ_CMD_IQSTART_F |
3334                                 FW_LEN16(c));
3335        c.type_to_iqandstindex = htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP) |
3336                FW_IQ_CMD_IQASYNCH_V(fwevtq) | FW_IQ_CMD_VIID_V(pi->viid) |
3337                FW_IQ_CMD_IQANDST_V(intr_idx < 0) |
3338                FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X) |
3339                FW_IQ_CMD_IQANDSTINDEX_V(intr_idx >= 0 ? intr_idx :
3340                                                        -intr_idx - 1));
3341        c.iqdroprss_to_iqesize = htons(FW_IQ_CMD_IQPCIECH_V(pi->tx_chan) |
3342                FW_IQ_CMD_IQGTSMODE_F |
3343                FW_IQ_CMD_IQINTCNTTHRESH_V(iq->pktcnt_idx) |
3344                FW_IQ_CMD_IQESIZE_V(ilog2(iq->iqe_len) - 4));
3345        c.iqsize = htons(iq->size);
3346        c.iqaddr = cpu_to_be64(iq->phys_addr);
3347        if (cong >= 0)
3348                c.iqns_to_fl0congen = htonl(FW_IQ_CMD_IQFLINTCONGEN_F |
3349                                FW_IQ_CMD_IQTYPE_V(cong ? FW_IQ_IQTYPE_NIC
3350                                                        :  FW_IQ_IQTYPE_OFLD));
3351
3352        if (fl) {
3353                enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);
3354
3355                /* Allocate the ring for the hardware free list (with space
3356                 * for its status page) along with the associated software
3357                 * descriptor ring.  The free list size needs to be a multiple
3358                 * of the Egress Queue Unit and at least 2 Egress Units larger
3359                 * than the SGE's Egress Congrestion Threshold
3360                 * (fl_starve_thres - 1).
3361                 */
3362                if (fl->size < s->fl_starve_thres - 1 + 2 * 8)
3363                        fl->size = s->fl_starve_thres - 1 + 2 * 8;
3364                fl->size = roundup(fl->size, 8);
3365                fl->desc = alloc_ring(adap->pdev_dev, fl->size, sizeof(__be64),
3366                                      sizeof(struct rx_sw_desc), &fl->addr,
3367                                      &fl->sdesc, s->stat_len,
3368                                      dev_to_node(adap->pdev_dev));
3369                if (!fl->desc)
3370                        goto fl_nomem;
3371
3372                flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc);
3373                c.iqns_to_fl0congen |= htonl(FW_IQ_CMD_FL0PACKEN_F |
3374                                             FW_IQ_CMD_FL0FETCHRO_V(relaxed) |
3375                                             FW_IQ_CMD_FL0DATARO_V(relaxed) |
3376                                             FW_IQ_CMD_FL0PADEN_F);
3377                if (cong >= 0)
3378                        c.iqns_to_fl0congen |=
3379                                htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong) |
3380                                      FW_IQ_CMD_FL0CONGCIF_F |
3381                                      FW_IQ_CMD_FL0CONGEN_F);
3382                /* In T6, for egress queue type FL there is internal overhead
3383                 * of 16B for header going into FLM module.  Hence the maximum
3384                 * allowed burst size is 448 bytes.  For T4/T5, the hardware
3385                 * doesn't coalesce fetch requests if more than 64 bytes of
3386                 * Free List pointers are provided, so we use a 128-byte Fetch
3387                 * Burst Minimum there (T6 implements coalescing so we can use
3388                 * the smaller 64-byte value there).
3389                 */
3390                c.fl0dcaen_to_fl0cidxfthresh =
3391                        htons(FW_IQ_CMD_FL0FBMIN_V(chip <= CHELSIO_T5 ?
3392                                                   FETCHBURSTMIN_128B_X :
3393                                                   FETCHBURSTMIN_64B_X) |
3394                              FW_IQ_CMD_FL0FBMAX_V((chip <= CHELSIO_T5) ?
3395                                                   FETCHBURSTMAX_512B_X :
3396                                                   FETCHBURSTMAX_256B_X));
3397                c.fl0size = htons(flsz);
3398                c.fl0addr = cpu_to_be64(fl->addr);
3399        }
3400
3401        ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3402        if (ret)
3403                goto err;
3404
3405        netif_napi_add(dev, &iq->napi, napi_rx_handler, 64);
3406        iq->cur_desc = iq->desc;
3407        iq->cidx = 0;
3408        iq->gen = 1;
3409        iq->next_intr_params = iq->intr_params;
3410        iq->cntxt_id = ntohs(c.iqid);
3411        iq->abs_id = ntohs(c.physiqid);
3412        iq->bar2_addr = bar2_address(adap,
3413                                     iq->cntxt_id,
3414                                     T4_BAR2_QTYPE_INGRESS,
3415                                     &iq->bar2_qid);
3416        iq->size--;                           /* subtract status entry */
3417        iq->netdev = dev;
3418        iq->handler = hnd;
3419        iq->flush_handler = flush_hnd;
3420
3421        memset(&iq->lro_mgr, 0, sizeof(struct t4_lro_mgr));
3422        skb_queue_head_init(&iq->lro_mgr.lroq);
3423
3424        /* set offset to -1 to distinguish ingress queues without FL */
3425        iq->offset = fl ? 0 : -1;
3426
3427        adap->sge.ingr_map[iq->cntxt_id - adap->sge.ingr_start] = iq;
3428
3429        if (fl) {
3430                fl->cntxt_id = ntohs(c.fl0id);
3431                fl->avail = fl->pend_cred = 0;
3432                fl->pidx = fl->cidx = 0;
3433                fl->alloc_failed = fl->large_alloc_failed = fl->starving = 0;
3434                adap->sge.egr_map[fl->cntxt_id - adap->sge.egr_start] = fl;
3435
3436                /* Note, we must initialize the BAR2 Free List User Doorbell
3437                 * information before refilling the Free List!
3438                 */
3439                fl->bar2_addr = bar2_address(adap,
3440                                             fl->cntxt_id,
3441                                             T4_BAR2_QTYPE_EGRESS,
3442                                             &fl->bar2_qid);
3443                refill_fl(adap, fl, fl_cap(fl), GFP_KERNEL);
3444        }
3445
3446        /* For T5 and later we attempt to set up the Congestion Manager values
3447         * of the new RX Ethernet Queue.  This should really be handled by
3448         * firmware because it's more complex than any host driver wants to
3449         * get involved with and it's different per chip and this is almost
3450         * certainly wrong.  Firmware would be wrong as well, but it would be
3451         * a lot easier to fix in one place ...  For now we do something very
3452         * simple (and hopefully less wrong).
3453         */
3454        if (!is_t4(adap->params.chip) && cong >= 0) {
3455                u32 param, val, ch_map = 0;
3456                int i;
3457                u16 cng_ch_bits_log = adap->params.arch.cng_ch_bits_log;
3458
3459                param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
3460                         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
3461                         FW_PARAMS_PARAM_YZ_V(iq->cntxt_id));
3462                if (cong == 0) {
3463                        val = CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X);
3464                } else {
3465                        val =
3466                            CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X);
3467                        for (i = 0; i < 4; i++) {
3468                                if (cong & (1 << i))
3469                                        ch_map |= 1 << (i << cng_ch_bits_log);
3470                        }
3471                        val |= CONMCTXT_CNGCHMAP_V(ch_map);
3472                }
3473                ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
3474                                    &param, &val);
3475                if (ret)
3476                        dev_warn(adap->pdev_dev, "Failed to set Congestion"
3477                                 " Manager Context for Ingress Queue %d: %d\n",
3478                                 iq->cntxt_id, -ret);
3479        }
3480
3481        return 0;
3482
3483fl_nomem:
3484        ret = -ENOMEM;
3485err:
3486        if (iq->desc) {
3487                dma_free_coherent(adap->pdev_dev, iq->size * iq->iqe_len,
3488                                  iq->desc, iq->phys_addr);
3489                iq->desc = NULL;
3490        }
3491        if (fl && fl->desc) {
3492                kfree(fl->sdesc);
3493                fl->sdesc = NULL;
3494                dma_free_coherent(adap->pdev_dev, flsz * sizeof(struct tx_desc),
3495                                  fl->desc, fl->addr);
3496                fl->desc = NULL;
3497        }
3498        return ret;
3499}
3500
3501static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id)
3502{
3503        q->cntxt_id = id;
3504        q->bar2_addr = bar2_address(adap,
3505                                    q->cntxt_id,
3506                                    T4_BAR2_QTYPE_EGRESS,
3507                                    &q->bar2_qid);
3508        q->in_use = 0;
3509        q->cidx = q->pidx = 0;
3510        q->stops = q->restarts = 0;
3511        q->stat = (void *)&q->desc[q->size];
3512        spin_lock_init(&q->db_lock);
3513        adap->sge.egr_map[id - adap->sge.egr_start] = q;
3514}
3515
3516int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
3517                         struct net_device *dev, struct netdev_queue *netdevq,
3518                         unsigned int iqid)
3519{
3520        int ret, nentries;
3521        struct fw_eq_eth_cmd c;
3522        struct sge *s = &adap->sge;
3523        struct port_info *pi = netdev_priv(dev);
3524
3525        /* Add status entries */
3526        nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3527
3528        txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
3529                        sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
3530                        &txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
3531                        netdev_queue_numa_node_read(netdevq));
3532        if (!txq->q.desc)
3533                return -ENOMEM;
3534
3535        memset(&c, 0, sizeof(c));
3536        c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD) | FW_CMD_REQUEST_F |
3537                            FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3538                            FW_EQ_ETH_CMD_PFN_V(adap->pf) |
3539                            FW_EQ_ETH_CMD_VFN_V(0));
3540        c.alloc_to_len16 = htonl(FW_EQ_ETH_CMD_ALLOC_F |
3541                                 FW_EQ_ETH_CMD_EQSTART_F | FW_LEN16(c));
3542        c.viid_pkd = htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F |
3543                           FW_EQ_ETH_CMD_VIID_V(pi->viid));
3544        c.fetchszm_to_iqid =
3545                htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3546                      FW_EQ_ETH_CMD_PCIECHN_V(pi->tx_chan) |
3547                      FW_EQ_ETH_CMD_FETCHRO_F | FW_EQ_ETH_CMD_IQID_V(iqid));
3548        c.dcaen_to_eqsize =
3549                htonl(FW_EQ_ETH_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
3550                      FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3551                      FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3552                      FW_EQ_ETH_CMD_EQSIZE_V(nentries));
3553        c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3554
3555        ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3556        if (ret) {
3557                kfree(txq->q.sdesc);
3558                txq->q.sdesc = NULL;
3559                dma_free_coherent(adap->pdev_dev,
3560                                  nentries * sizeof(struct tx_desc),
3561                                  txq->q.desc, txq->q.phys_addr);
3562                txq->q.desc = NULL;
3563                return ret;
3564        }
3565
3566        txq->q.q_type = CXGB4_TXQ_ETH;
3567        init_txq(adap, &txq->q, FW_EQ_ETH_CMD_EQID_G(ntohl(c.eqid_pkd)));
3568        txq->txq = netdevq;
3569        txq->tso = txq->tx_cso = txq->vlan_ins = 0;
3570        txq->mapping_err = 0;
3571        return 0;
3572}
3573
3574int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
3575                          struct net_device *dev, unsigned int iqid,
3576                          unsigned int cmplqid)
3577{
3578        int ret, nentries;
3579        struct fw_eq_ctrl_cmd c;
3580        struct sge *s = &adap->sge;
3581        struct port_info *pi = netdev_priv(dev);
3582
3583        /* Add status entries */
3584        nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3585
3586        txq->q.desc = alloc_ring(adap->pdev_dev, nentries,
3587                                 sizeof(struct tx_desc), 0, &txq->q.phys_addr,
3588                                 NULL, 0, dev_to_node(adap->pdev_dev));
3589        if (!txq->q.desc)
3590                return -ENOMEM;
3591
3592        c.op_to_vfn = htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD) | FW_CMD_REQUEST_F |
3593                            FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3594                            FW_EQ_CTRL_CMD_PFN_V(adap->pf) |
3595                            FW_EQ_CTRL_CMD_VFN_V(0));
3596        c.alloc_to_len16 = htonl(FW_EQ_CTRL_CMD_ALLOC_F |
3597                                 FW_EQ_CTRL_CMD_EQSTART_F | FW_LEN16(c));
3598        c.cmpliqid_eqid = htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid));
3599        c.physeqid_pkd = htonl(0);
3600        c.fetchszm_to_iqid =
3601                htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3602                      FW_EQ_CTRL_CMD_PCIECHN_V(pi->tx_chan) |
3603                      FW_EQ_CTRL_CMD_FETCHRO_F | FW_EQ_CTRL_CMD_IQID_V(iqid));
3604        c.dcaen_to_eqsize =
3605                htonl(FW_EQ_CTRL_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
3606                      FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3607                      FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3608                      FW_EQ_CTRL_CMD_EQSIZE_V(nentries));
3609        c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3610
3611        ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3612        if (ret) {
3613                dma_free_coherent(adap->pdev_dev,
3614                                  nentries * sizeof(struct tx_desc),
3615                                  txq->q.desc, txq->q.phys_addr);
3616                txq->q.desc = NULL;
3617                return ret;
3618        }
3619
3620        txq->q.q_type = CXGB4_TXQ_CTRL;
3621        init_txq(adap, &txq->q, FW_EQ_CTRL_CMD_EQID_G(ntohl(c.cmpliqid_eqid)));
3622        txq->adap = adap;
3623        skb_queue_head_init(&txq->sendq);
3624        tasklet_init(&txq->qresume_tsk, restart_ctrlq, (unsigned long)txq);
3625        txq->full = 0;
3626        return 0;
3627}
3628
3629int t4_sge_mod_ctrl_txq(struct adapter *adap, unsigned int eqid,
3630                        unsigned int cmplqid)
3631{
3632        u32 param, val;
3633
3634        param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
3635                 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL) |
3636                 FW_PARAMS_PARAM_YZ_V(eqid));
3637        val = cmplqid;
3638        return t4_set_params(adap, adap->mbox, adap->pf, 0, 1, &param, &val);
3639}
3640
3641int t4_sge_alloc_uld_txq(struct adapter *adap, struct sge_uld_txq *txq,
3642                         struct net_device *dev, unsigned int iqid,
3643                         unsigned int uld_type)
3644{
3645        int ret, nentries;
3646        struct fw_eq_ofld_cmd c;
3647        struct sge *s = &adap->sge;
3648        struct port_info *pi = netdev_priv(dev);
3649        int cmd = FW_EQ_OFLD_CMD;
3650
3651        /* Add status entries */
3652        nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
3653
3654        txq->q.desc = alloc_ring(adap->pdev_dev, txq->q.size,
3655                        sizeof(struct tx_desc), sizeof(struct tx_sw_desc),
3656                        &txq->q.phys_addr, &txq->q.sdesc, s->stat_len,
3657                        NUMA_NO_NODE);
3658        if (!txq->q.desc)
3659                return -ENOMEM;
3660
3661        memset(&c, 0, sizeof(c));
3662        if (unlikely(uld_type == CXGB4_TX_CRYPTO))
3663                cmd = FW_EQ_CTRL_CMD;
3664        c.op_to_vfn = htonl(FW_CMD_OP_V(cmd) | FW_CMD_REQUEST_F |
3665                            FW_CMD_WRITE_F | FW_CMD_EXEC_F |
3666                            FW_EQ_OFLD_CMD_PFN_V(adap->pf) |
3667                            FW_EQ_OFLD_CMD_VFN_V(0));
3668        c.alloc_to_len16 = htonl(FW_EQ_OFLD_CMD_ALLOC_F |
3669                                 FW_EQ_OFLD_CMD_EQSTART_F | FW_LEN16(c));
3670        c.fetchszm_to_iqid =
3671                htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X) |
3672                      FW_EQ_OFLD_CMD_PCIECHN_V(pi->tx_chan) |
3673                      FW_EQ_OFLD_CMD_FETCHRO_F | FW_EQ_OFLD_CMD_IQID_V(iqid));
3674        c.dcaen_to_eqsize =
3675                htonl(FW_EQ_OFLD_CMD_FBMIN_V(FETCHBURSTMIN_64B_X) |
3676                      FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X) |
3677                      FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X) |
3678                      FW_EQ_OFLD_CMD_EQSIZE_V(nentries));
3679        c.eqaddr = cpu_to_be64(txq->q.phys_addr);
3680
3681        ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
3682        if (ret) {
3683                kfree(txq->q.sdesc);
3684                txq->q.sdesc = NULL;
3685                dma_free_coherent(adap->pdev_dev,
3686                                  nentries * sizeof(struct tx_desc),
3687                                  txq->q.desc, txq->q.phys_addr);
3688                txq->q.desc = NULL;
3689                return ret;
3690        }
3691
3692        txq->q.q_type = CXGB4_TXQ_ULD;
3693        init_txq(adap, &txq->q, FW_EQ_OFLD_CMD_EQID_G(ntohl(c.eqid_pkd)));
3694        txq->adap = adap;
3695        skb_queue_head_init(&txq->sendq);
3696        tasklet_init(&txq->qresume_tsk, restart_ofldq, (unsigned long)txq);
3697        txq->full = 0;
3698        txq->mapping_err = 0;
3699        return 0;
3700}
3701
3702void free_txq(struct adapter *adap, struct sge_txq *q)
3703{
3704        struct sge *s = &adap->sge;
3705
3706        dma_free_coherent(adap->pdev_dev,
3707                          q->size * sizeof(struct tx_desc) + s->stat_len,
3708                          q->desc, q->phys_addr);
3709        q->cntxt_id = 0;
3710        q->sdesc = NULL;
3711        q->desc = NULL;
3712}
3713
3714void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
3715                  struct sge_fl *fl)
3716{
3717        struct sge *s = &adap->sge;
3718        unsigned int fl_id = fl ? fl->cntxt_id : 0xffff;
3719
3720        adap->sge.ingr_map[rq->cntxt_id - adap->sge.ingr_start] = NULL;
3721        t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
3722                   rq->cntxt_id, fl_id, 0xffff);
3723        dma_free_coherent(adap->pdev_dev, (rq->size + 1) * rq->iqe_len,
3724                          rq->desc, rq->phys_addr);
3725        netif_napi_del(&rq->napi);
3726        rq->netdev = NULL;
3727        rq->cntxt_id = rq->abs_id = 0;
3728        rq->desc = NULL;
3729
3730        if (fl) {
3731                free_rx_bufs(adap, fl, fl->avail);
3732                dma_free_coherent(adap->pdev_dev, fl->size * 8 + s->stat_len,
3733                                  fl->desc, fl->addr);
3734                kfree(fl->sdesc);
3735                fl->sdesc = NULL;
3736                fl->cntxt_id = 0;
3737                fl->desc = NULL;
3738        }
3739}
3740
3741/**
3742 *      t4_free_ofld_rxqs - free a block of consecutive Rx queues
3743 *      @adap: the adapter
3744 *      @n: number of queues
3745 *      @q: pointer to first queue
3746 *
3747 *      Release the resources of a consecutive block of offload Rx queues.
3748 */
3749void t4_free_ofld_rxqs(struct adapter *adap, int n, struct sge_ofld_rxq *q)
3750{
3751        for ( ; n; n--, q++)
3752                if (q->rspq.desc)
3753                        free_rspq_fl(adap, &q->rspq,
3754                                     q->fl.size ? &q->fl : NULL);
3755}
3756
3757/**
3758 *      t4_free_sge_resources - free SGE resources
3759 *      @adap: the adapter
3760 *
3761 *      Frees resources used by the SGE queue sets.
3762 */
3763void t4_free_sge_resources(struct adapter *adap)
3764{
3765        int i;
3766        struct sge_eth_rxq *eq;
3767        struct sge_eth_txq *etq;
3768
3769        /* stop all Rx queues in order to start them draining */
3770        for (i = 0; i < adap->sge.ethqsets; i++) {
3771                eq = &adap->sge.ethrxq[i];
3772                if (eq->rspq.desc)
3773                        t4_iq_stop(adap, adap->mbox, adap->pf, 0,
3774                                   FW_IQ_TYPE_FL_INT_CAP,
3775                                   eq->rspq.cntxt_id,
3776                                   eq->fl.size ? eq->fl.cntxt_id : 0xffff,
3777                                   0xffff);
3778        }
3779
3780        /* clean up Ethernet Tx/Rx queues */
3781        for (i = 0; i < adap->sge.ethqsets; i++) {
3782                eq = &adap->sge.ethrxq[i];
3783                if (eq->rspq.desc)
3784                        free_rspq_fl(adap, &eq->rspq,
3785                                     eq->fl.size ? &eq->fl : NULL);
3786
3787                etq = &adap->sge.ethtxq[i];
3788                if (etq->q.desc) {
3789                        t4_eth_eq_free(adap, adap->mbox, adap->pf, 0,
3790                                       etq->q.cntxt_id);
3791                        __netif_tx_lock_bh(etq->txq);
3792                        free_tx_desc(adap, &etq->q, etq->q.in_use, true);
3793                        __netif_tx_unlock_bh(etq->txq);
3794                        kfree(etq->q.sdesc);
3795                        free_txq(adap, &etq->q);
3796                }
3797        }
3798
3799        /* clean up control Tx queues */
3800        for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) {
3801                struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i];
3802
3803                if (cq->q.desc) {
3804                        tasklet_kill(&cq->qresume_tsk);
3805                        t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0,
3806                                        cq->q.cntxt_id);
3807                        __skb_queue_purge(&cq->sendq);
3808                        free_txq(adap, &cq->q);
3809                }
3810        }
3811
3812        if (adap->sge.fw_evtq.desc)
3813                free_rspq_fl(adap, &adap->sge.fw_evtq, NULL);
3814
3815        if (adap->sge.intrq.desc)
3816                free_rspq_fl(adap, &adap->sge.intrq, NULL);
3817
3818        if (!is_t4(adap->params.chip)) {
3819                etq = &adap->sge.ptptxq;
3820                if (etq->q.desc) {
3821                        t4_eth_eq_free(adap, adap->mbox, adap->pf, 0,
3822                                       etq->q.cntxt_id);
3823                        spin_lock_bh(&adap->ptp_lock);
3824                        free_tx_desc(adap, &etq->q, etq->q.in_use, true);
3825                        spin_unlock_bh(&adap->ptp_lock);
3826                        kfree(etq->q.sdesc);
3827                        free_txq(adap, &etq->q);
3828                }
3829        }
3830
3831        /* clear the reverse egress queue map */
3832        memset(adap->sge.egr_map, 0,
3833               adap->sge.egr_sz * sizeof(*adap->sge.egr_map));
3834}
3835
3836void t4_sge_start(struct adapter *adap)
3837{
3838        adap->sge.ethtxq_rover = 0;
3839        mod_timer(&adap->sge.rx_timer, jiffies + RX_QCHECK_PERIOD);
3840        mod_timer(&adap->sge.tx_timer, jiffies + TX_QCHECK_PERIOD);
3841}
3842
3843/**
3844 *      t4_sge_stop - disable SGE operation
3845 *      @adap: the adapter
3846 *
3847 *      Stop tasklets and timers associated with the DMA engine.  Note that
3848 *      this is effective only if measures have been taken to disable any HW
3849 *      events that may restart them.
3850 */
3851void t4_sge_stop(struct adapter *adap)
3852{
3853        int i;
3854        struct sge *s = &adap->sge;
3855
3856        if (in_interrupt())  /* actions below require waiting */
3857                return;
3858
3859        if (s->rx_timer.function)
3860                del_timer_sync(&s->rx_timer);
3861        if (s->tx_timer.function)
3862                del_timer_sync(&s->tx_timer);
3863
3864        if (is_offload(adap)) {
3865                struct sge_uld_txq_info *txq_info;
3866
3867                txq_info = adap->sge.uld_txq_info[CXGB4_TX_OFLD];
3868                if (txq_info) {
3869                        struct sge_uld_txq *txq = txq_info->uldtxq;
3870
3871                        for_each_ofldtxq(&adap->sge, i) {
3872                                if (txq->q.desc)
3873                                        tasklet_kill(&txq->qresume_tsk);
3874                        }
3875                }
3876        }
3877
3878        if (is_pci_uld(adap)) {
3879                struct sge_uld_txq_info *txq_info;
3880
3881                txq_info = adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
3882                if (txq_info) {
3883                        struct sge_uld_txq *txq = txq_info->uldtxq;
3884
3885                        for_each_ofldtxq(&adap->sge, i) {
3886                                if (txq->q.desc)
3887                                        tasklet_kill(&txq->qresume_tsk);
3888                        }
3889                }
3890        }
3891
3892        for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) {
3893                struct sge_ctrl_txq *cq = &s->ctrlq[i];
3894
3895                if (cq->q.desc)
3896                        tasklet_kill(&cq->qresume_tsk);
3897        }
3898}
3899
3900/**
3901 *      t4_sge_init_soft - grab core SGE values needed by SGE code
3902 *      @adap: the adapter
3903 *
3904 *      We need to grab the SGE operating parameters that we need to have
3905 *      in order to do our job and make sure we can live with them.
3906 */
3907
3908static int t4_sge_init_soft(struct adapter *adap)
3909{
3910        struct sge *s = &adap->sge;
3911        u32 fl_small_pg, fl_large_pg, fl_small_mtu, fl_large_mtu;
3912        u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
3913        u32 ingress_rx_threshold;
3914
3915        /*
3916         * Verify that CPL messages are going to the Ingress Queue for
3917         * process_responses() and that only packet data is going to the
3918         * Free Lists.
3919         */
3920        if ((t4_read_reg(adap, SGE_CONTROL_A) & RXPKTCPLMODE_F) !=
3921            RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X)) {
3922                dev_err(adap->pdev_dev, "bad SGE CPL MODE\n");
3923                return -EINVAL;
3924        }
3925
3926        /*
3927         * Validate the Host Buffer Register Array indices that we want to
3928         * use ...
3929         *
3930         * XXX Note that we should really read through the Host Buffer Size
3931         * XXX register array and find the indices of the Buffer Sizes which
3932         * XXX meet our needs!
3933         */
3934        #define READ_FL_BUF(x) \
3935                t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32))
3936
3937        fl_small_pg = READ_FL_BUF(RX_SMALL_PG_BUF);
3938        fl_large_pg = READ_FL_BUF(RX_LARGE_PG_BUF);
3939        fl_small_mtu = READ_FL_BUF(RX_SMALL_MTU_BUF);
3940        fl_large_mtu = READ_FL_BUF(RX_LARGE_MTU_BUF);
3941
3942        /* We only bother using the Large Page logic if the Large Page Buffer
3943         * is larger than our Page Size Buffer.
3944         */
3945        if (fl_large_pg <= fl_small_pg)
3946                fl_large_pg = 0;
3947
3948        #undef READ_FL_BUF
3949
3950        /* The Page Size Buffer must be exactly equal to our Page Size and the
3951         * Large Page Size Buffer should be 0 (per above) or a power of 2.
3952         */
3953        if (fl_small_pg != PAGE_SIZE ||
3954            (fl_large_pg & (fl_large_pg-1)) != 0) {
3955                dev_err(adap->pdev_dev, "bad SGE FL page buffer sizes [%d, %d]\n",
3956                        fl_small_pg, fl_large_pg);
3957                return -EINVAL;
3958        }
3959        if (fl_large_pg)
3960                s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
3961
3962        if (fl_small_mtu < FL_MTU_SMALL_BUFSIZE(adap) ||
3963            fl_large_mtu < FL_MTU_LARGE_BUFSIZE(adap)) {
3964                dev_err(adap->pdev_dev, "bad SGE FL MTU sizes [%d, %d]\n",
3965                        fl_small_mtu, fl_large_mtu);
3966                return -EINVAL;
3967        }
3968
3969        /*
3970         * Retrieve our RX interrupt holdoff timer values and counter
3971         * threshold values from the SGE parameters.
3972         */
3973        timer_value_0_and_1 = t4_read_reg(adap, SGE_TIMER_VALUE_0_AND_1_A);
3974        timer_value_2_and_3 = t4_read_reg(adap, SGE_TIMER_VALUE_2_AND_3_A);
3975        timer_value_4_and_5 = t4_read_reg(adap, SGE_TIMER_VALUE_4_AND_5_A);
3976        s->timer_val[0] = core_ticks_to_us(adap,
3977                TIMERVALUE0_G(timer_value_0_and_1));
3978        s->timer_val[1] = core_ticks_to_us(adap,
3979                TIMERVALUE1_G(timer_value_0_and_1));
3980        s->timer_val[2] = core_ticks_to_us(adap,
3981                TIMERVALUE2_G(timer_value_2_and_3));
3982        s->timer_val[3] = core_ticks_to_us(adap,
3983                TIMERVALUE3_G(timer_value_2_and_3));
3984        s->timer_val[4] = core_ticks_to_us(adap,
3985                TIMERVALUE4_G(timer_value_4_and_5));
3986        s->timer_val[5] = core_ticks_to_us(adap,
3987                TIMERVALUE5_G(timer_value_4_and_5));
3988
3989        ingress_rx_threshold = t4_read_reg(adap, SGE_INGRESS_RX_THRESHOLD_A);
3990        s->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
3991        s->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
3992        s->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
3993        s->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
3994
3995        return 0;
3996}
3997
3998/**
3999 *     t4_sge_init - initialize SGE
4000 *     @adap: the adapter
4001 *
4002 *     Perform low-level SGE code initialization needed every time after a
4003 *     chip reset.
4004 */
4005int t4_sge_init(struct adapter *adap)
4006{
4007        struct sge *s = &adap->sge;
4008        u32 sge_control, sge_conm_ctrl;
4009        int ret, egress_threshold;
4010
4011        /*
4012         * Ingress Padding Boundary and Egress Status Page Size are set up by
4013         * t4_fixup_host_params().
4014         */
4015        sge_control = t4_read_reg(adap, SGE_CONTROL_A);
4016        s->pktshift = PKTSHIFT_G(sge_control);
4017        s->stat_len = (sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64;
4018
4019        s->fl_align = t4_fl_pkt_align(adap);
4020        ret = t4_sge_init_soft(adap);
4021        if (ret < 0)
4022                return ret;
4023
4024        /*
4025         * A FL with <= fl_starve_thres buffers is starving and a periodic
4026         * timer will attempt to refill it.  This needs to be larger than the
4027         * SGE's Egress Congestion Threshold.  If it isn't, then we can get
4028         * stuck waiting for new packets while the SGE is waiting for us to
4029         * give it more Free List entries.  (Note that the SGE's Egress
4030         * Congestion Threshold is in units of 2 Free List pointers.) For T4,
4031         * there was only a single field to control this.  For T5 there's the
4032         * original field which now only applies to Unpacked Mode Free List
4033         * buffers and a new field which only applies to Packed Mode Free List
4034         * buffers.
4035         */
4036        sge_conm_ctrl = t4_read_reg(adap, SGE_CONM_CTRL_A);
4037        switch (CHELSIO_CHIP_VERSION(adap->params.chip)) {
4038        case CHELSIO_T4:
4039                egress_threshold = EGRTHRESHOLD_G(sge_conm_ctrl);
4040                break;
4041        case CHELSIO_T5:
4042                egress_threshold = EGRTHRESHOLDPACKING_G(sge_conm_ctrl);
4043                break;
4044        case CHELSIO_T6:
4045                egress_threshold = T6_EGRTHRESHOLDPACKING_G(sge_conm_ctrl);
4046                break;
4047        default:
4048                dev_err(adap->pdev_dev, "Unsupported Chip version %d\n",
4049                        CHELSIO_CHIP_VERSION(adap->params.chip));
4050                return -EINVAL;
4051        }
4052        s->fl_starve_thres = 2*egress_threshold + 1;
4053
4054        t4_idma_monitor_init(adap, &s->idma_monitor);
4055
4056        /* Set up timers used for recuring callbacks to process RX and TX
4057         * administrative tasks.
4058         */
4059        timer_setup(&s->rx_timer, sge_rx_timer_cb, 0);
4060        timer_setup(&s->tx_timer, sge_tx_timer_cb, 0);
4061
4062        spin_lock_init(&s->intrq_lock);
4063
4064        return 0;
4065}
4066