linux/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
<<
>>
Prefs
   1/* bnx2x_cmn.c: QLogic Everest network driver.
   2 *
   3 * Copyright (c) 2007-2013 Broadcom Corporation
   4 * Copyright (c) 2014 QLogic Corporation
   5 * All rights reserved
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation.
  10 *
  11 * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
  12 * Written by: Eliezer Tamir
  13 * Based on code from Michael Chan's bnx2 driver
  14 * UDP CSUM errata workaround by Arik Gendelman
  15 * Slowpath and fastpath rework by Vladislav Zolotarov
  16 * Statistics and Link management by Yitchak Gertner
  17 *
  18 */
  19
  20#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  21
  22#include <linux/etherdevice.h>
  23#include <linux/if_vlan.h>
  24#include <linux/interrupt.h>
  25#include <linux/ip.h>
  26#include <linux/crash_dump.h>
  27#include <net/tcp.h>
  28#include <net/ipv6.h>
  29#include <net/ip6_checksum.h>
  30#include <net/busy_poll.h>
  31#include <linux/prefetch.h>
  32#include "bnx2x_cmn.h"
  33#include "bnx2x_init.h"
  34#include "bnx2x_sp.h"
  35
  36static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
  37static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
  38static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
  39static int bnx2x_poll(struct napi_struct *napi, int budget);
  40
  41static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
  42{
  43        int i;
  44
  45        /* Add NAPI objects */
  46        for_each_rx_queue_cnic(bp, i) {
  47                netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  48                               bnx2x_poll, NAPI_POLL_WEIGHT);
  49        }
  50}
  51
  52static void bnx2x_add_all_napi(struct bnx2x *bp)
  53{
  54        int i;
  55
  56        /* Add NAPI objects */
  57        for_each_eth_queue(bp, i) {
  58                netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
  59                               bnx2x_poll, NAPI_POLL_WEIGHT);
  60        }
  61}
  62
  63static int bnx2x_calc_num_queues(struct bnx2x *bp)
  64{
  65        int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
  66
  67        /* Reduce memory usage in kdump environment by using only one queue */
  68        if (is_kdump_kernel())
  69                nq = 1;
  70
  71        nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
  72        return nq;
  73}
  74
  75/**
  76 * bnx2x_move_fp - move content of the fastpath structure.
  77 *
  78 * @bp:         driver handle
  79 * @from:       source FP index
  80 * @to:         destination FP index
  81 *
  82 * Makes sure the contents of the bp->fp[to].napi is kept
  83 * intact. This is done by first copying the napi struct from
  84 * the target to the source, and then mem copying the entire
  85 * source onto the target. Update txdata pointers and related
  86 * content.
  87 */
  88static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  89{
  90        struct bnx2x_fastpath *from_fp = &bp->fp[from];
  91        struct bnx2x_fastpath *to_fp = &bp->fp[to];
  92        struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  93        struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  94        struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  95        struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  96        int old_max_eth_txqs, new_max_eth_txqs;
  97        int old_txdata_index = 0, new_txdata_index = 0;
  98        struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
  99
 100        /* Copy the NAPI object as it has been already initialized */
 101        from_fp->napi = to_fp->napi;
 102
 103        /* Move bnx2x_fastpath contents */
 104        memcpy(to_fp, from_fp, sizeof(*to_fp));
 105        to_fp->index = to;
 106
 107        /* Retain the tpa_info of the original `to' version as we don't want
 108         * 2 FPs to contain the same tpa_info pointer.
 109         */
 110        to_fp->tpa_info = old_tpa_info;
 111
 112        /* move sp_objs contents as well, as their indices match fp ones */
 113        memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
 114
 115        /* move fp_stats contents as well, as their indices match fp ones */
 116        memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
 117
 118        /* Update txdata pointers in fp and move txdata content accordingly:
 119         * Each fp consumes 'max_cos' txdata structures, so the index should be
 120         * decremented by max_cos x delta.
 121         */
 122
 123        old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
 124        new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
 125                                (bp)->max_cos;
 126        if (from == FCOE_IDX(bp)) {
 127                old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 128                new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
 129        }
 130
 131        memcpy(&bp->bnx2x_txq[new_txdata_index],
 132               &bp->bnx2x_txq[old_txdata_index],
 133               sizeof(struct bnx2x_fp_txdata));
 134        to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
 135}
 136
 137/**
 138 * bnx2x_fill_fw_str - Fill buffer with FW version string.
 139 *
 140 * @bp:        driver handle
 141 * @buf:       character buffer to fill with the fw name
 142 * @buf_len:   length of the above buffer
 143 *
 144 */
 145void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 146{
 147        if (IS_PF(bp)) {
 148                u8 phy_fw_ver[PHY_FW_VER_LEN];
 149
 150                phy_fw_ver[0] = '\0';
 151                bnx2x_get_ext_phy_fw_version(&bp->link_params,
 152                                             phy_fw_ver, PHY_FW_VER_LEN);
 153                strlcpy(buf, bp->fw_ver, buf_len);
 154                snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 155                         "bc %d.%d.%d%s%s",
 156                         (bp->common.bc_ver & 0xff0000) >> 16,
 157                         (bp->common.bc_ver & 0xff00) >> 8,
 158                         (bp->common.bc_ver & 0xff),
 159                         ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 160        } else {
 161                bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 162        }
 163}
 164
 165/**
 166 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 167 *
 168 * @bp: driver handle
 169 * @delta:      number of eth queues which were not allocated
 170 */
 171static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 172{
 173        int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 174
 175        /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 176         * backward along the array could cause memory to be overridden
 177         */
 178        for (cos = 1; cos < bp->max_cos; cos++) {
 179                for (i = 0; i < old_eth_num - delta; i++) {
 180                        struct bnx2x_fastpath *fp = &bp->fp[i];
 181                        int new_idx = cos * (old_eth_num - delta) + i;
 182
 183                        memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 184                               sizeof(struct bnx2x_fp_txdata));
 185                        fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 186                }
 187        }
 188}
 189
 190int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 191
 192/* free skb in the packet ring at pos idx
 193 * return idx of last bd freed
 194 */
 195static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 196                             u16 idx, unsigned int *pkts_compl,
 197                             unsigned int *bytes_compl)
 198{
 199        struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 200        struct eth_tx_start_bd *tx_start_bd;
 201        struct eth_tx_bd *tx_data_bd;
 202        struct sk_buff *skb = tx_buf->skb;
 203        u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 204        int nbd;
 205        u16 split_bd_len = 0;
 206
 207        /* prefetch skb end pointer to speedup dev_kfree_skb() */
 208        prefetch(&skb->end);
 209
 210        DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 211           txdata->txq_index, idx, tx_buf, skb);
 212
 213        tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 214
 215        nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 216#ifdef BNX2X_STOP_ON_ERROR
 217        if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 218                BNX2X_ERR("BAD nbd!\n");
 219                bnx2x_panic();
 220        }
 221#endif
 222        new_cons = nbd + tx_buf->first_bd;
 223
 224        /* Get the next bd */
 225        bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 226
 227        /* Skip a parse bd... */
 228        --nbd;
 229        bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 230
 231        if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
 232                /* Skip second parse bd... */
 233                --nbd;
 234                bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 235        }
 236
 237        /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
 238        if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 239                tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 240                split_bd_len = BD_UNMAP_LEN(tx_data_bd);
 241                --nbd;
 242                bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 243        }
 244
 245        /* unmap first bd */
 246        dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 247                         BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
 248                         DMA_TO_DEVICE);
 249
 250        /* now free frags */
 251        while (nbd > 0) {
 252
 253                tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 254                dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 255                               BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 256                if (--nbd)
 257                        bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 258        }
 259
 260        /* release skb */
 261        WARN_ON(!skb);
 262        if (likely(skb)) {
 263                (*pkts_compl)++;
 264                (*bytes_compl) += skb->len;
 265                dev_kfree_skb_any(skb);
 266        }
 267
 268        tx_buf->first_bd = 0;
 269        tx_buf->skb = NULL;
 270
 271        return new_cons;
 272}
 273
 274int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 275{
 276        struct netdev_queue *txq;
 277        u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 278        unsigned int pkts_compl = 0, bytes_compl = 0;
 279
 280#ifdef BNX2X_STOP_ON_ERROR
 281        if (unlikely(bp->panic))
 282                return -1;
 283#endif
 284
 285        txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 286        hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 287        sw_cons = txdata->tx_pkt_cons;
 288
 289        while (sw_cons != hw_cons) {
 290                u16 pkt_cons;
 291
 292                pkt_cons = TX_BD(sw_cons);
 293
 294                DP(NETIF_MSG_TX_DONE,
 295                   "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 296                   txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 297
 298                bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 299                                            &pkts_compl, &bytes_compl);
 300
 301                sw_cons++;
 302        }
 303
 304        netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 305
 306        txdata->tx_pkt_cons = sw_cons;
 307        txdata->tx_bd_cons = bd_cons;
 308
 309        /* Need to make the tx_bd_cons update visible to start_xmit()
 310         * before checking for netif_tx_queue_stopped().  Without the
 311         * memory barrier, there is a small possibility that
 312         * start_xmit() will miss it and cause the queue to be stopped
 313         * forever.
 314         * On the other hand we need an rmb() here to ensure the proper
 315         * ordering of bit testing in the following
 316         * netif_tx_queue_stopped(txq) call.
 317         */
 318        smp_mb();
 319
 320        if (unlikely(netif_tx_queue_stopped(txq))) {
 321                /* Taking tx_lock() is needed to prevent re-enabling the queue
 322                 * while it's empty. This could have happen if rx_action() gets
 323                 * suspended in bnx2x_tx_int() after the condition before
 324                 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 325                 *
 326                 * stops the queue->sees fresh tx_bd_cons->releases the queue->
 327                 * sends some packets consuming the whole queue again->
 328                 * stops the queue
 329                 */
 330
 331                __netif_tx_lock(txq, smp_processor_id());
 332
 333                if ((netif_tx_queue_stopped(txq)) &&
 334                    (bp->state == BNX2X_STATE_OPEN) &&
 335                    (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 336                        netif_tx_wake_queue(txq);
 337
 338                __netif_tx_unlock(txq);
 339        }
 340        return 0;
 341}
 342
 343static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 344                                             u16 idx)
 345{
 346        u16 last_max = fp->last_max_sge;
 347
 348        if (SUB_S16(idx, last_max) > 0)
 349                fp->last_max_sge = idx;
 350}
 351
 352static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 353                                         u16 sge_len,
 354                                         struct eth_end_agg_rx_cqe *cqe)
 355{
 356        struct bnx2x *bp = fp->bp;
 357        u16 last_max, last_elem, first_elem;
 358        u16 delta = 0;
 359        u16 i;
 360
 361        if (!sge_len)
 362                return;
 363
 364        /* First mark all used pages */
 365        for (i = 0; i < sge_len; i++)
 366                BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 367                        RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 368
 369        DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 370           sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 371
 372        /* Here we assume that the last SGE index is the biggest */
 373        prefetch((void *)(fp->sge_mask));
 374        bnx2x_update_last_max_sge(fp,
 375                le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 376
 377        last_max = RX_SGE(fp->last_max_sge);
 378        last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 379        first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 380
 381        /* If ring is not full */
 382        if (last_elem + 1 != first_elem)
 383                last_elem++;
 384
 385        /* Now update the prod */
 386        for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 387                if (likely(fp->sge_mask[i]))
 388                        break;
 389
 390                fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 391                delta += BIT_VEC64_ELEM_SZ;
 392        }
 393
 394        if (delta > 0) {
 395                fp->rx_sge_prod += delta;
 396                /* clear page-end entries */
 397                bnx2x_clear_sge_mask_next_elems(fp);
 398        }
 399
 400        DP(NETIF_MSG_RX_STATUS,
 401           "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 402           fp->last_max_sge, fp->rx_sge_prod);
 403}
 404
 405/* Get Toeplitz hash value in the skb using the value from the
 406 * CQE (calculated by HW).
 407 */
 408static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 409                            const struct eth_fast_path_rx_cqe *cqe,
 410                            enum pkt_hash_types *rxhash_type)
 411{
 412        /* Get Toeplitz hash from CQE */
 413        if ((bp->dev->features & NETIF_F_RXHASH) &&
 414            (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 415                enum eth_rss_hash_type htype;
 416
 417                htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 418                *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
 419                                (htype == TCP_IPV6_HASH_TYPE)) ?
 420                               PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
 421
 422                return le32_to_cpu(cqe->rss_hash_result);
 423        }
 424        *rxhash_type = PKT_HASH_TYPE_NONE;
 425        return 0;
 426}
 427
 428static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 429                            u16 cons, u16 prod,
 430                            struct eth_fast_path_rx_cqe *cqe)
 431{
 432        struct bnx2x *bp = fp->bp;
 433        struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 434        struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 435        struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 436        dma_addr_t mapping;
 437        struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 438        struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 439
 440        /* print error if current state != stop */
 441        if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 442                BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 443
 444        /* Try to map an empty data buffer from the aggregation info  */
 445        mapping = dma_map_single(&bp->pdev->dev,
 446                                 first_buf->data + NET_SKB_PAD,
 447                                 fp->rx_buf_size, DMA_FROM_DEVICE);
 448        /*
 449         *  ...if it fails - move the skb from the consumer to the producer
 450         *  and set the current aggregation state as ERROR to drop it
 451         *  when TPA_STOP arrives.
 452         */
 453
 454        if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 455                /* Move the BD from the consumer to the producer */
 456                bnx2x_reuse_rx_data(fp, cons, prod);
 457                tpa_info->tpa_state = BNX2X_TPA_ERROR;
 458                return;
 459        }
 460
 461        /* move empty data from pool to prod */
 462        prod_rx_buf->data = first_buf->data;
 463        dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 464        /* point prod_bd to new data */
 465        prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 466        prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 467
 468        /* move partial skb from cons to pool (don't unmap yet) */
 469        *first_buf = *cons_rx_buf;
 470
 471        /* mark bin state as START */
 472        tpa_info->parsing_flags =
 473                le16_to_cpu(cqe->pars_flags.flags);
 474        tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 475        tpa_info->tpa_state = BNX2X_TPA_START;
 476        tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 477        tpa_info->placement_offset = cqe->placement_offset;
 478        tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
 479        if (fp->mode == TPA_MODE_GRO) {
 480                u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 481                tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 482                tpa_info->gro_size = gro_size;
 483        }
 484
 485#ifdef BNX2X_STOP_ON_ERROR
 486        fp->tpa_queue_used |= (1 << queue);
 487        DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 488           fp->tpa_queue_used);
 489#endif
 490}
 491
 492/* Timestamp option length allowed for TPA aggregation:
 493 *
 494 *              nop nop kind length echo val
 495 */
 496#define TPA_TSTAMP_OPT_LEN      12
 497/**
 498 * bnx2x_set_gro_params - compute GRO values
 499 *
 500 * @skb:                packet skb
 501 * @parsing_flags:      parsing flags from the START CQE
 502 * @len_on_bd:          total length of the first packet for the
 503 *                      aggregation.
 504 * @pkt_len:            length of all segments
 505 *
 506 * Approximate value of the MSS for this aggregation calculated using
 507 * the first packet of it.
 508 * Compute number of aggregated segments, and gso_type.
 509 */
 510static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 511                                 u16 len_on_bd, unsigned int pkt_len,
 512                                 u16 num_of_coalesced_segs)
 513{
 514        /* TPA aggregation won't have either IP options or TCP options
 515         * other than timestamp or IPv6 extension headers.
 516         */
 517        u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 518
 519        if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 520            PRS_FLAG_OVERETH_IPV6) {
 521                hdrs_len += sizeof(struct ipv6hdr);
 522                skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 523        } else {
 524                hdrs_len += sizeof(struct iphdr);
 525                skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 526        }
 527
 528        /* Check if there was a TCP timestamp, if there is it's will
 529         * always be 12 bytes length: nop nop kind length echo val.
 530         *
 531         * Otherwise FW would close the aggregation.
 532         */
 533        if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 534                hdrs_len += TPA_TSTAMP_OPT_LEN;
 535
 536        skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 537
 538        /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 539         * to skb_shinfo(skb)->gso_segs
 540         */
 541        NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 542}
 543
 544static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 545                              u16 index, gfp_t gfp_mask)
 546{
 547        struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 548        struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 549        struct bnx2x_alloc_pool *pool = &fp->page_pool;
 550        dma_addr_t mapping;
 551
 552        if (!pool->page) {
 553                pool->page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
 554                if (unlikely(!pool->page))
 555                        return -ENOMEM;
 556
 557                pool->offset = 0;
 558        }
 559
 560        mapping = dma_map_page(&bp->pdev->dev, pool->page,
 561                               pool->offset, SGE_PAGE_SIZE, DMA_FROM_DEVICE);
 562        if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 563                BNX2X_ERR("Can't map sge\n");
 564                return -ENOMEM;
 565        }
 566
 567        sw_buf->page = pool->page;
 568        sw_buf->offset = pool->offset;
 569
 570        dma_unmap_addr_set(sw_buf, mapping, mapping);
 571
 572        sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 573        sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 574
 575        pool->offset += SGE_PAGE_SIZE;
 576        if (PAGE_SIZE - pool->offset >= SGE_PAGE_SIZE)
 577                get_page(pool->page);
 578        else
 579                pool->page = NULL;
 580        return 0;
 581}
 582
 583static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 584                               struct bnx2x_agg_info *tpa_info,
 585                               u16 pages,
 586                               struct sk_buff *skb,
 587                               struct eth_end_agg_rx_cqe *cqe,
 588                               u16 cqe_idx)
 589{
 590        struct sw_rx_page *rx_pg, old_rx_pg;
 591        u32 i, frag_len, frag_size;
 592        int err, j, frag_id = 0;
 593        u16 len_on_bd = tpa_info->len_on_bd;
 594        u16 full_page = 0, gro_size = 0;
 595
 596        frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 597
 598        if (fp->mode == TPA_MODE_GRO) {
 599                gro_size = tpa_info->gro_size;
 600                full_page = tpa_info->full_page;
 601        }
 602
 603        /* This is needed in order to enable forwarding support */
 604        if (frag_size)
 605                bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 606                                     le16_to_cpu(cqe->pkt_len),
 607                                     le16_to_cpu(cqe->num_of_coalesced_segs));
 608
 609#ifdef BNX2X_STOP_ON_ERROR
 610        if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 611                BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 612                          pages, cqe_idx);
 613                BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 614                bnx2x_panic();
 615                return -EINVAL;
 616        }
 617#endif
 618
 619        /* Run through the SGL and compose the fragmented skb */
 620        for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 621                u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 622
 623                /* FW gives the indices of the SGE as if the ring is an array
 624                   (meaning that "next" element will consume 2 indices) */
 625                if (fp->mode == TPA_MODE_GRO)
 626                        frag_len = min_t(u32, frag_size, (u32)full_page);
 627                else /* LRO */
 628                        frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 629
 630                rx_pg = &fp->rx_page_ring[sge_idx];
 631                old_rx_pg = *rx_pg;
 632
 633                /* If we fail to allocate a substitute page, we simply stop
 634                   where we are and drop the whole packet */
 635                err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
 636                if (unlikely(err)) {
 637                        bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 638                        return err;
 639                }
 640
 641                dma_unmap_page(&bp->pdev->dev,
 642                               dma_unmap_addr(&old_rx_pg, mapping),
 643                               SGE_PAGE_SIZE, DMA_FROM_DEVICE);
 644                /* Add one frag and update the appropriate fields in the skb */
 645                if (fp->mode == TPA_MODE_LRO)
 646                        skb_fill_page_desc(skb, j, old_rx_pg.page,
 647                                           old_rx_pg.offset, frag_len);
 648                else { /* GRO */
 649                        int rem;
 650                        int offset = 0;
 651                        for (rem = frag_len; rem > 0; rem -= gro_size) {
 652                                int len = rem > gro_size ? gro_size : rem;
 653                                skb_fill_page_desc(skb, frag_id++,
 654                                                   old_rx_pg.page,
 655                                                   old_rx_pg.offset + offset,
 656                                                   len);
 657                                if (offset)
 658                                        get_page(old_rx_pg.page);
 659                                offset += len;
 660                        }
 661                }
 662
 663                skb->data_len += frag_len;
 664                skb->truesize += SGE_PAGES;
 665                skb->len += frag_len;
 666
 667                frag_size -= frag_len;
 668        }
 669
 670        return 0;
 671}
 672
 673static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 674{
 675        if (fp->rx_frag_size)
 676                skb_free_frag(data);
 677        else
 678                kfree(data);
 679}
 680
 681static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
 682{
 683        if (fp->rx_frag_size) {
 684                /* GFP_KERNEL allocations are used only during initialization */
 685                if (unlikely(gfpflags_allow_blocking(gfp_mask)))
 686                        return (void *)__get_free_page(gfp_mask);
 687
 688                return netdev_alloc_frag(fp->rx_frag_size);
 689        }
 690
 691        return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
 692}
 693
 694#ifdef CONFIG_INET
 695static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 696{
 697        const struct iphdr *iph = ip_hdr(skb);
 698        struct tcphdr *th;
 699
 700        skb_set_transport_header(skb, sizeof(struct iphdr));
 701        th = tcp_hdr(skb);
 702
 703        th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 704                                  iph->saddr, iph->daddr, 0);
 705}
 706
 707static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 708{
 709        struct ipv6hdr *iph = ipv6_hdr(skb);
 710        struct tcphdr *th;
 711
 712        skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 713        th = tcp_hdr(skb);
 714
 715        th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 716                                  &iph->saddr, &iph->daddr, 0);
 717}
 718
 719static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 720                            void (*gro_func)(struct bnx2x*, struct sk_buff*))
 721{
 722        skb_reset_network_header(skb);
 723        gro_func(bp, skb);
 724        tcp_gro_complete(skb);
 725}
 726#endif
 727
 728static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 729                               struct sk_buff *skb)
 730{
 731#ifdef CONFIG_INET
 732        if (skb_shinfo(skb)->gso_size) {
 733                switch (be16_to_cpu(skb->protocol)) {
 734                case ETH_P_IP:
 735                        bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 736                        break;
 737                case ETH_P_IPV6:
 738                        bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 739                        break;
 740                default:
 741                        WARN_ONCE(1, "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 742                                  be16_to_cpu(skb->protocol));
 743                }
 744        }
 745#endif
 746        skb_record_rx_queue(skb, fp->rx_queue);
 747        napi_gro_receive(&fp->napi, skb);
 748}
 749
 750static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 751                           struct bnx2x_agg_info *tpa_info,
 752                           u16 pages,
 753                           struct eth_end_agg_rx_cqe *cqe,
 754                           u16 cqe_idx)
 755{
 756        struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 757        u8 pad = tpa_info->placement_offset;
 758        u16 len = tpa_info->len_on_bd;
 759        struct sk_buff *skb = NULL;
 760        u8 *new_data, *data = rx_buf->data;
 761        u8 old_tpa_state = tpa_info->tpa_state;
 762
 763        tpa_info->tpa_state = BNX2X_TPA_STOP;
 764
 765        /* If we there was an error during the handling of the TPA_START -
 766         * drop this aggregation.
 767         */
 768        if (old_tpa_state == BNX2X_TPA_ERROR)
 769                goto drop;
 770
 771        /* Try to allocate the new data */
 772        new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
 773        /* Unmap skb in the pool anyway, as we are going to change
 774           pool entry status to BNX2X_TPA_STOP even if new skb allocation
 775           fails. */
 776        dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 777                         fp->rx_buf_size, DMA_FROM_DEVICE);
 778        if (likely(new_data))
 779                skb = build_skb(data, fp->rx_frag_size);
 780
 781        if (likely(skb)) {
 782#ifdef BNX2X_STOP_ON_ERROR
 783                if (pad + len > fp->rx_buf_size) {
 784                        BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 785                                  pad, len, fp->rx_buf_size);
 786                        bnx2x_panic();
 787                        return;
 788                }
 789#endif
 790
 791                skb_reserve(skb, pad + NET_SKB_PAD);
 792                skb_put(skb, len);
 793                skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
 794
 795                skb->protocol = eth_type_trans(skb, bp->dev);
 796                skb->ip_summed = CHECKSUM_UNNECESSARY;
 797
 798                if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 799                                         skb, cqe, cqe_idx)) {
 800                        if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 801                                __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 802                        bnx2x_gro_receive(bp, fp, skb);
 803                } else {
 804                        DP(NETIF_MSG_RX_STATUS,
 805                           "Failed to allocate new pages - dropping packet!\n");
 806                        dev_kfree_skb_any(skb);
 807                }
 808
 809                /* put new data in bin */
 810                rx_buf->data = new_data;
 811
 812                return;
 813        }
 814        if (new_data)
 815                bnx2x_frag_free(fp, new_data);
 816drop:
 817        /* drop the packet and keep the buffer in the bin */
 818        DP(NETIF_MSG_RX_STATUS,
 819           "Failed to allocate or map a new skb - dropping packet!\n");
 820        bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 821}
 822
 823static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 824                               u16 index, gfp_t gfp_mask)
 825{
 826        u8 *data;
 827        struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 828        struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 829        dma_addr_t mapping;
 830
 831        data = bnx2x_frag_alloc(fp, gfp_mask);
 832        if (unlikely(data == NULL))
 833                return -ENOMEM;
 834
 835        mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 836                                 fp->rx_buf_size,
 837                                 DMA_FROM_DEVICE);
 838        if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 839                bnx2x_frag_free(fp, data);
 840                BNX2X_ERR("Can't map rx data\n");
 841                return -ENOMEM;
 842        }
 843
 844        rx_buf->data = data;
 845        dma_unmap_addr_set(rx_buf, mapping, mapping);
 846
 847        rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 848        rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 849
 850        return 0;
 851}
 852
 853static
 854void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 855                                 struct bnx2x_fastpath *fp,
 856                                 struct bnx2x_eth_q_stats *qstats)
 857{
 858        /* Do nothing if no L4 csum validation was done.
 859         * We do not check whether IP csum was validated. For IPv4 we assume
 860         * that if the card got as far as validating the L4 csum, it also
 861         * validated the IP csum. IPv6 has no IP csum.
 862         */
 863        if (cqe->fast_path_cqe.status_flags &
 864            ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 865                return;
 866
 867        /* If L4 validation was done, check if an error was found. */
 868
 869        if (cqe->fast_path_cqe.type_error_flags &
 870            (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 871             ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 872                qstats->hw_csum_err++;
 873        else
 874                skb->ip_summed = CHECKSUM_UNNECESSARY;
 875}
 876
 877static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 878{
 879        struct bnx2x *bp = fp->bp;
 880        u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 881        u16 sw_comp_cons, sw_comp_prod;
 882        int rx_pkt = 0;
 883        union eth_rx_cqe *cqe;
 884        struct eth_fast_path_rx_cqe *cqe_fp;
 885
 886#ifdef BNX2X_STOP_ON_ERROR
 887        if (unlikely(bp->panic))
 888                return 0;
 889#endif
 890        if (budget <= 0)
 891                return rx_pkt;
 892
 893        bd_cons = fp->rx_bd_cons;
 894        bd_prod = fp->rx_bd_prod;
 895        bd_prod_fw = bd_prod;
 896        sw_comp_cons = fp->rx_comp_cons;
 897        sw_comp_prod = fp->rx_comp_prod;
 898
 899        comp_ring_cons = RCQ_BD(sw_comp_cons);
 900        cqe = &fp->rx_comp_ring[comp_ring_cons];
 901        cqe_fp = &cqe->fast_path_cqe;
 902
 903        DP(NETIF_MSG_RX_STATUS,
 904           "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
 905
 906        while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
 907                struct sw_rx_bd *rx_buf = NULL;
 908                struct sk_buff *skb;
 909                u8 cqe_fp_flags;
 910                enum eth_rx_cqe_type cqe_fp_type;
 911                u16 len, pad, queue;
 912                u8 *data;
 913                u32 rxhash;
 914                enum pkt_hash_types rxhash_type;
 915
 916#ifdef BNX2X_STOP_ON_ERROR
 917                if (unlikely(bp->panic))
 918                        return 0;
 919#endif
 920
 921                bd_prod = RX_BD(bd_prod);
 922                bd_cons = RX_BD(bd_cons);
 923
 924                /* A rmb() is required to ensure that the CQE is not read
 925                 * before it is written by the adapter DMA.  PCI ordering
 926                 * rules will make sure the other fields are written before
 927                 * the marker at the end of struct eth_fast_path_rx_cqe
 928                 * but without rmb() a weakly ordered processor can process
 929                 * stale data.  Without the barrier TPA state-machine might
 930                 * enter inconsistent state and kernel stack might be
 931                 * provided with incorrect packet description - these lead
 932                 * to various kernel crashed.
 933                 */
 934                rmb();
 935
 936                cqe_fp_flags = cqe_fp->type_error_flags;
 937                cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 938
 939                DP(NETIF_MSG_RX_STATUS,
 940                   "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 941                   CQE_TYPE(cqe_fp_flags),
 942                   cqe_fp_flags, cqe_fp->status_flags,
 943                   le32_to_cpu(cqe_fp->rss_hash_result),
 944                   le16_to_cpu(cqe_fp->vlan_tag),
 945                   le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 946
 947                /* is this a slowpath msg? */
 948                if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 949                        bnx2x_sp_event(fp, cqe);
 950                        goto next_cqe;
 951                }
 952
 953                rx_buf = &fp->rx_buf_ring[bd_cons];
 954                data = rx_buf->data;
 955
 956                if (!CQE_TYPE_FAST(cqe_fp_type)) {
 957                        struct bnx2x_agg_info *tpa_info;
 958                        u16 frag_size, pages;
 959#ifdef BNX2X_STOP_ON_ERROR
 960                        /* sanity check */
 961                        if (fp->mode == TPA_MODE_DISABLED &&
 962                            (CQE_TYPE_START(cqe_fp_type) ||
 963                             CQE_TYPE_STOP(cqe_fp_type)))
 964                                BNX2X_ERR("START/STOP packet while TPA disabled, type %x\n",
 965                                          CQE_TYPE(cqe_fp_type));
 966#endif
 967
 968                        if (CQE_TYPE_START(cqe_fp_type)) {
 969                                u16 queue = cqe_fp->queue_index;
 970                                DP(NETIF_MSG_RX_STATUS,
 971                                   "calling tpa_start on queue %d\n",
 972                                   queue);
 973
 974                                bnx2x_tpa_start(fp, queue,
 975                                                bd_cons, bd_prod,
 976                                                cqe_fp);
 977
 978                                goto next_rx;
 979                        }
 980                        queue = cqe->end_agg_cqe.queue_index;
 981                        tpa_info = &fp->tpa_info[queue];
 982                        DP(NETIF_MSG_RX_STATUS,
 983                           "calling tpa_stop on queue %d\n",
 984                           queue);
 985
 986                        frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 987                                    tpa_info->len_on_bd;
 988
 989                        if (fp->mode == TPA_MODE_GRO)
 990                                pages = (frag_size + tpa_info->full_page - 1) /
 991                                         tpa_info->full_page;
 992                        else
 993                                pages = SGE_PAGE_ALIGN(frag_size) >>
 994                                        SGE_PAGE_SHIFT;
 995
 996                        bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 997                                       &cqe->end_agg_cqe, comp_ring_cons);
 998#ifdef BNX2X_STOP_ON_ERROR
 999                        if (bp->panic)
1000                                return 0;
1001#endif
1002
1003                        bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
1004                        goto next_cqe;
1005                }
1006                /* non TPA */
1007                len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
1008                pad = cqe_fp->placement_offset;
1009                dma_sync_single_for_cpu(&bp->pdev->dev,
1010                                        dma_unmap_addr(rx_buf, mapping),
1011                                        pad + RX_COPY_THRESH,
1012                                        DMA_FROM_DEVICE);
1013                pad += NET_SKB_PAD;
1014                prefetch(data + pad); /* speedup eth_type_trans() */
1015                /* is this an error packet? */
1016                if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
1017                        DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1018                           "ERROR  flags %x  rx packet %u\n",
1019                           cqe_fp_flags, sw_comp_cons);
1020                        bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1021                        goto reuse_rx;
1022                }
1023
1024                /* Since we don't have a jumbo ring
1025                 * copy small packets if mtu > 1500
1026                 */
1027                if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1028                    (len <= RX_COPY_THRESH)) {
1029                        skb = napi_alloc_skb(&fp->napi, len);
1030                        if (skb == NULL) {
1031                                DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1032                                   "ERROR  packet dropped because of alloc failure\n");
1033                                bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1034                                goto reuse_rx;
1035                        }
1036                        memcpy(skb->data, data + pad, len);
1037                        bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1038                } else {
1039                        if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1040                                                       GFP_ATOMIC) == 0)) {
1041                                dma_unmap_single(&bp->pdev->dev,
1042                                                 dma_unmap_addr(rx_buf, mapping),
1043                                                 fp->rx_buf_size,
1044                                                 DMA_FROM_DEVICE);
1045                                skb = build_skb(data, fp->rx_frag_size);
1046                                if (unlikely(!skb)) {
1047                                        bnx2x_frag_free(fp, data);
1048                                        bnx2x_fp_qstats(bp, fp)->
1049                                                        rx_skb_alloc_failed++;
1050                                        goto next_rx;
1051                                }
1052                                skb_reserve(skb, pad);
1053                        } else {
1054                                DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1055                                   "ERROR  packet dropped because of alloc failure\n");
1056                                bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1057reuse_rx:
1058                                bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1059                                goto next_rx;
1060                        }
1061                }
1062
1063                skb_put(skb, len);
1064                skb->protocol = eth_type_trans(skb, bp->dev);
1065
1066                /* Set Toeplitz hash for a none-LRO skb */
1067                rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1068                skb_set_hash(skb, rxhash, rxhash_type);
1069
1070                skb_checksum_none_assert(skb);
1071
1072                if (bp->dev->features & NETIF_F_RXCSUM)
1073                        bnx2x_csum_validate(skb, cqe, fp,
1074                                            bnx2x_fp_qstats(bp, fp));
1075
1076                skb_record_rx_queue(skb, fp->rx_queue);
1077
1078                /* Check if this packet was timestamped */
1079                if (unlikely(cqe->fast_path_cqe.type_error_flags &
1080                             (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1081                        bnx2x_set_rx_ts(bp, skb);
1082
1083                if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1084                    PARSING_FLAGS_VLAN)
1085                        __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1086                                               le16_to_cpu(cqe_fp->vlan_tag));
1087
1088                napi_gro_receive(&fp->napi, skb);
1089next_rx:
1090                rx_buf->data = NULL;
1091
1092                bd_cons = NEXT_RX_IDX(bd_cons);
1093                bd_prod = NEXT_RX_IDX(bd_prod);
1094                bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1095                rx_pkt++;
1096next_cqe:
1097                sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1098                sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1099
1100                /* mark CQE as free */
1101                BNX2X_SEED_CQE(cqe_fp);
1102
1103                if (rx_pkt == budget)
1104                        break;
1105
1106                comp_ring_cons = RCQ_BD(sw_comp_cons);
1107                cqe = &fp->rx_comp_ring[comp_ring_cons];
1108                cqe_fp = &cqe->fast_path_cqe;
1109        } /* while */
1110
1111        fp->rx_bd_cons = bd_cons;
1112        fp->rx_bd_prod = bd_prod_fw;
1113        fp->rx_comp_cons = sw_comp_cons;
1114        fp->rx_comp_prod = sw_comp_prod;
1115
1116        /* Update producers */
1117        bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1118                             fp->rx_sge_prod);
1119
1120        return rx_pkt;
1121}
1122
1123static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1124{
1125        struct bnx2x_fastpath *fp = fp_cookie;
1126        struct bnx2x *bp = fp->bp;
1127        u8 cos;
1128
1129        DP(NETIF_MSG_INTR,
1130           "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1131           fp->index, fp->fw_sb_id, fp->igu_sb_id);
1132
1133        bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1134
1135#ifdef BNX2X_STOP_ON_ERROR
1136        if (unlikely(bp->panic))
1137                return IRQ_HANDLED;
1138#endif
1139
1140        /* Handle Rx and Tx according to MSI-X vector */
1141        for_each_cos_in_tx_queue(fp, cos)
1142                prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1143
1144        prefetch(&fp->sb_running_index[SM_RX_ID]);
1145        napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1146
1147        return IRQ_HANDLED;
1148}
1149
1150/* HW Lock for shared dual port PHYs */
1151void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1152{
1153        mutex_lock(&bp->port.phy_mutex);
1154
1155        bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1156}
1157
1158void bnx2x_release_phy_lock(struct bnx2x *bp)
1159{
1160        bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1161
1162        mutex_unlock(&bp->port.phy_mutex);
1163}
1164
1165/* calculates MF speed according to current linespeed and MF configuration */
1166u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1167{
1168        u16 line_speed = bp->link_vars.line_speed;
1169        if (IS_MF(bp)) {
1170                u16 maxCfg = bnx2x_extract_max_cfg(bp,
1171                                                   bp->mf_config[BP_VN(bp)]);
1172
1173                /* Calculate the current MAX line speed limit for the MF
1174                 * devices
1175                 */
1176                if (IS_MF_PERCENT_BW(bp))
1177                        line_speed = (line_speed * maxCfg) / 100;
1178                else { /* SD mode */
1179                        u16 vn_max_rate = maxCfg * 100;
1180
1181                        if (vn_max_rate < line_speed)
1182                                line_speed = vn_max_rate;
1183                }
1184        }
1185
1186        return line_speed;
1187}
1188
1189/**
1190 * bnx2x_fill_report_data - fill link report data to report
1191 *
1192 * @bp:         driver handle
1193 * @data:       link state to update
1194 *
1195 * It uses a none-atomic bit operations because is called under the mutex.
1196 */
1197static void bnx2x_fill_report_data(struct bnx2x *bp,
1198                                   struct bnx2x_link_report_data *data)
1199{
1200        memset(data, 0, sizeof(*data));
1201
1202        if (IS_PF(bp)) {
1203                /* Fill the report data: effective line speed */
1204                data->line_speed = bnx2x_get_mf_speed(bp);
1205
1206                /* Link is down */
1207                if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1208                        __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1209                                  &data->link_report_flags);
1210
1211                if (!BNX2X_NUM_ETH_QUEUES(bp))
1212                        __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1213                                  &data->link_report_flags);
1214
1215                /* Full DUPLEX */
1216                if (bp->link_vars.duplex == DUPLEX_FULL)
1217                        __set_bit(BNX2X_LINK_REPORT_FD,
1218                                  &data->link_report_flags);
1219
1220                /* Rx Flow Control is ON */
1221                if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1222                        __set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1223                                  &data->link_report_flags);
1224
1225                /* Tx Flow Control is ON */
1226                if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1227                        __set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1228                                  &data->link_report_flags);
1229        } else { /* VF */
1230                *data = bp->vf_link_vars;
1231        }
1232}
1233
1234/**
1235 * bnx2x_link_report - report link status to OS.
1236 *
1237 * @bp:         driver handle
1238 *
1239 * Calls the __bnx2x_link_report() under the same locking scheme
1240 * as a link/PHY state managing code to ensure a consistent link
1241 * reporting.
1242 */
1243
1244void bnx2x_link_report(struct bnx2x *bp)
1245{
1246        bnx2x_acquire_phy_lock(bp);
1247        __bnx2x_link_report(bp);
1248        bnx2x_release_phy_lock(bp);
1249}
1250
1251/**
1252 * __bnx2x_link_report - report link status to OS.
1253 *
1254 * @bp:         driver handle
1255 *
1256 * None atomic implementation.
1257 * Should be called under the phy_lock.
1258 */
1259void __bnx2x_link_report(struct bnx2x *bp)
1260{
1261        struct bnx2x_link_report_data cur_data;
1262
1263        /* reread mf_cfg */
1264        if (IS_PF(bp) && !CHIP_IS_E1(bp))
1265                bnx2x_read_mf_cfg(bp);
1266
1267        /* Read the current link report info */
1268        bnx2x_fill_report_data(bp, &cur_data);
1269
1270        /* Don't report link down or exactly the same link status twice */
1271        if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1272            (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1273                      &bp->last_reported_link.link_report_flags) &&
1274             test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1275                      &cur_data.link_report_flags)))
1276                return;
1277
1278        bp->link_cnt++;
1279
1280        /* We are going to report a new link parameters now -
1281         * remember the current data for the next time.
1282         */
1283        memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1284
1285        /* propagate status to VFs */
1286        if (IS_PF(bp))
1287                bnx2x_iov_link_update(bp);
1288
1289        if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1290                     &cur_data.link_report_flags)) {
1291                netif_carrier_off(bp->dev);
1292                netdev_err(bp->dev, "NIC Link is Down\n");
1293                return;
1294        } else {
1295                const char *duplex;
1296                const char *flow;
1297
1298                netif_carrier_on(bp->dev);
1299
1300                if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1301                                       &cur_data.link_report_flags))
1302                        duplex = "full";
1303                else
1304                        duplex = "half";
1305
1306                /* Handle the FC at the end so that only these flags would be
1307                 * possibly set. This way we may easily check if there is no FC
1308                 * enabled.
1309                 */
1310                if (cur_data.link_report_flags) {
1311                        if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1312                                     &cur_data.link_report_flags)) {
1313                                if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1314                                     &cur_data.link_report_flags))
1315                                        flow = "ON - receive & transmit";
1316                                else
1317                                        flow = "ON - receive";
1318                        } else {
1319                                flow = "ON - transmit";
1320                        }
1321                } else {
1322                        flow = "none";
1323                }
1324                netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1325                            cur_data.line_speed, duplex, flow);
1326        }
1327}
1328
1329static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1330{
1331        int i;
1332
1333        for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1334                struct eth_rx_sge *sge;
1335
1336                sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1337                sge->addr_hi =
1338                        cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1339                        BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1340
1341                sge->addr_lo =
1342                        cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1343                        BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1344        }
1345}
1346
1347static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1348                                struct bnx2x_fastpath *fp, int last)
1349{
1350        int i;
1351
1352        for (i = 0; i < last; i++) {
1353                struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1354                struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1355                u8 *data = first_buf->data;
1356
1357                if (data == NULL) {
1358                        DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1359                        continue;
1360                }
1361                if (tpa_info->tpa_state == BNX2X_TPA_START)
1362                        dma_unmap_single(&bp->pdev->dev,
1363                                         dma_unmap_addr(first_buf, mapping),
1364                                         fp->rx_buf_size, DMA_FROM_DEVICE);
1365                bnx2x_frag_free(fp, data);
1366                first_buf->data = NULL;
1367        }
1368}
1369
1370void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1371{
1372        int j;
1373
1374        for_each_rx_queue_cnic(bp, j) {
1375                struct bnx2x_fastpath *fp = &bp->fp[j];
1376
1377                fp->rx_bd_cons = 0;
1378
1379                /* Activate BD ring */
1380                /* Warning!
1381                 * this will generate an interrupt (to the TSTORM)
1382                 * must only be done after chip is initialized
1383                 */
1384                bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1385                                     fp->rx_sge_prod);
1386        }
1387}
1388
1389void bnx2x_init_rx_rings(struct bnx2x *bp)
1390{
1391        int func = BP_FUNC(bp);
1392        u16 ring_prod;
1393        int i, j;
1394
1395        /* Allocate TPA resources */
1396        for_each_eth_queue(bp, j) {
1397                struct bnx2x_fastpath *fp = &bp->fp[j];
1398
1399                DP(NETIF_MSG_IFUP,
1400                   "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1401
1402                if (fp->mode != TPA_MODE_DISABLED) {
1403                        /* Fill the per-aggregation pool */
1404                        for (i = 0; i < MAX_AGG_QS(bp); i++) {
1405                                struct bnx2x_agg_info *tpa_info =
1406                                        &fp->tpa_info[i];
1407                                struct sw_rx_bd *first_buf =
1408                                        &tpa_info->first_buf;
1409
1410                                first_buf->data =
1411                                        bnx2x_frag_alloc(fp, GFP_KERNEL);
1412                                if (!first_buf->data) {
1413                                        BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1414                                                  j);
1415                                        bnx2x_free_tpa_pool(bp, fp, i);
1416                                        fp->mode = TPA_MODE_DISABLED;
1417                                        break;
1418                                }
1419                                dma_unmap_addr_set(first_buf, mapping, 0);
1420                                tpa_info->tpa_state = BNX2X_TPA_STOP;
1421                        }
1422
1423                        /* "next page" elements initialization */
1424                        bnx2x_set_next_page_sgl(fp);
1425
1426                        /* set SGEs bit mask */
1427                        bnx2x_init_sge_ring_bit_mask(fp);
1428
1429                        /* Allocate SGEs and initialize the ring elements */
1430                        for (i = 0, ring_prod = 0;
1431                             i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1432
1433                                if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1434                                                       GFP_KERNEL) < 0) {
1435                                        BNX2X_ERR("was only able to allocate %d rx sges\n",
1436                                                  i);
1437                                        BNX2X_ERR("disabling TPA for queue[%d]\n",
1438                                                  j);
1439                                        /* Cleanup already allocated elements */
1440                                        bnx2x_free_rx_sge_range(bp, fp,
1441                                                                ring_prod);
1442                                        bnx2x_free_tpa_pool(bp, fp,
1443                                                            MAX_AGG_QS(bp));
1444                                        fp->mode = TPA_MODE_DISABLED;
1445                                        ring_prod = 0;
1446                                        break;
1447                                }
1448                                ring_prod = NEXT_SGE_IDX(ring_prod);
1449                        }
1450
1451                        fp->rx_sge_prod = ring_prod;
1452                }
1453        }
1454
1455        for_each_eth_queue(bp, j) {
1456                struct bnx2x_fastpath *fp = &bp->fp[j];
1457
1458                fp->rx_bd_cons = 0;
1459
1460                /* Activate BD ring */
1461                /* Warning!
1462                 * this will generate an interrupt (to the TSTORM)
1463                 * must only be done after chip is initialized
1464                 */
1465                bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1466                                     fp->rx_sge_prod);
1467
1468                if (j != 0)
1469                        continue;
1470
1471                if (CHIP_IS_E1(bp)) {
1472                        REG_WR(bp, BAR_USTRORM_INTMEM +
1473                               USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1474                               U64_LO(fp->rx_comp_mapping));
1475                        REG_WR(bp, BAR_USTRORM_INTMEM +
1476                               USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1477                               U64_HI(fp->rx_comp_mapping));
1478                }
1479        }
1480}
1481
1482static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1483{
1484        u8 cos;
1485        struct bnx2x *bp = fp->bp;
1486
1487        for_each_cos_in_tx_queue(fp, cos) {
1488                struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1489                unsigned pkts_compl = 0, bytes_compl = 0;
1490
1491                u16 sw_prod = txdata->tx_pkt_prod;
1492                u16 sw_cons = txdata->tx_pkt_cons;
1493
1494                while (sw_cons != sw_prod) {
1495                        bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1496                                          &pkts_compl, &bytes_compl);
1497                        sw_cons++;
1498                }
1499
1500                netdev_tx_reset_queue(
1501                        netdev_get_tx_queue(bp->dev,
1502                                            txdata->txq_index));
1503        }
1504}
1505
1506static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1507{
1508        int i;
1509
1510        for_each_tx_queue_cnic(bp, i) {
1511                bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1512        }
1513}
1514
1515static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1516{
1517        int i;
1518
1519        for_each_eth_queue(bp, i) {
1520                bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1521        }
1522}
1523
1524static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1525{
1526        struct bnx2x *bp = fp->bp;
1527        int i;
1528
1529        /* ring wasn't allocated */
1530        if (fp->rx_buf_ring == NULL)
1531                return;
1532
1533        for (i = 0; i < NUM_RX_BD; i++) {
1534                struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1535                u8 *data = rx_buf->data;
1536
1537                if (data == NULL)
1538                        continue;
1539                dma_unmap_single(&bp->pdev->dev,
1540                                 dma_unmap_addr(rx_buf, mapping),
1541                                 fp->rx_buf_size, DMA_FROM_DEVICE);
1542
1543                rx_buf->data = NULL;
1544                bnx2x_frag_free(fp, data);
1545        }
1546}
1547
1548static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1549{
1550        int j;
1551
1552        for_each_rx_queue_cnic(bp, j) {
1553                bnx2x_free_rx_bds(&bp->fp[j]);
1554        }
1555}
1556
1557static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1558{
1559        int j;
1560
1561        for_each_eth_queue(bp, j) {
1562                struct bnx2x_fastpath *fp = &bp->fp[j];
1563
1564                bnx2x_free_rx_bds(fp);
1565
1566                if (fp->mode != TPA_MODE_DISABLED)
1567                        bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1568        }
1569}
1570
1571static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1572{
1573        bnx2x_free_tx_skbs_cnic(bp);
1574        bnx2x_free_rx_skbs_cnic(bp);
1575}
1576
1577void bnx2x_free_skbs(struct bnx2x *bp)
1578{
1579        bnx2x_free_tx_skbs(bp);
1580        bnx2x_free_rx_skbs(bp);
1581}
1582
1583void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1584{
1585        /* load old values */
1586        u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1587
1588        if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1589                /* leave all but MAX value */
1590                mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1591
1592                /* set new MAX value */
1593                mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1594                                & FUNC_MF_CFG_MAX_BW_MASK;
1595
1596                bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1597        }
1598}
1599
1600/**
1601 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1602 *
1603 * @bp:         driver handle
1604 * @nvecs:      number of vectors to be released
1605 */
1606static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1607{
1608        int i, offset = 0;
1609
1610        if (nvecs == offset)
1611                return;
1612
1613        /* VFs don't have a default SB */
1614        if (IS_PF(bp)) {
1615                free_irq(bp->msix_table[offset].vector, bp->dev);
1616                DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1617                   bp->msix_table[offset].vector);
1618                offset++;
1619        }
1620
1621        if (CNIC_SUPPORT(bp)) {
1622                if (nvecs == offset)
1623                        return;
1624                offset++;
1625        }
1626
1627        for_each_eth_queue(bp, i) {
1628                if (nvecs == offset)
1629                        return;
1630                DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1631                   i, bp->msix_table[offset].vector);
1632
1633                free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1634        }
1635}
1636
1637void bnx2x_free_irq(struct bnx2x *bp)
1638{
1639        if (bp->flags & USING_MSIX_FLAG &&
1640            !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1641                int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1642
1643                /* vfs don't have a default status block */
1644                if (IS_PF(bp))
1645                        nvecs++;
1646
1647                bnx2x_free_msix_irqs(bp, nvecs);
1648        } else {
1649                free_irq(bp->dev->irq, bp->dev);
1650        }
1651}
1652
1653int bnx2x_enable_msix(struct bnx2x *bp)
1654{
1655        int msix_vec = 0, i, rc;
1656
1657        /* VFs don't have a default status block */
1658        if (IS_PF(bp)) {
1659                bp->msix_table[msix_vec].entry = msix_vec;
1660                BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1661                               bp->msix_table[0].entry);
1662                msix_vec++;
1663        }
1664
1665        /* Cnic requires an msix vector for itself */
1666        if (CNIC_SUPPORT(bp)) {
1667                bp->msix_table[msix_vec].entry = msix_vec;
1668                BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1669                               msix_vec, bp->msix_table[msix_vec].entry);
1670                msix_vec++;
1671        }
1672
1673        /* We need separate vectors for ETH queues only (not FCoE) */
1674        for_each_eth_queue(bp, i) {
1675                bp->msix_table[msix_vec].entry = msix_vec;
1676                BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1677                               msix_vec, msix_vec, i);
1678                msix_vec++;
1679        }
1680
1681        DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1682           msix_vec);
1683
1684        rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1685                                   BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1686        /*
1687         * reconfigure number of tx/rx queues according to available
1688         * MSI-X vectors
1689         */
1690        if (rc == -ENOSPC) {
1691                /* Get by with single vector */
1692                rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1693                if (rc < 0) {
1694                        BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1695                                       rc);
1696                        goto no_msix;
1697                }
1698
1699                BNX2X_DEV_INFO("Using single MSI-X vector\n");
1700                bp->flags |= USING_SINGLE_MSIX_FLAG;
1701
1702                BNX2X_DEV_INFO("set number of queues to 1\n");
1703                bp->num_ethernet_queues = 1;
1704                bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1705        } else if (rc < 0) {
1706                BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1707                goto no_msix;
1708        } else if (rc < msix_vec) {
1709                /* how less vectors we will have? */
1710                int diff = msix_vec - rc;
1711
1712                BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1713
1714                /*
1715                 * decrease number of queues by number of unallocated entries
1716                 */
1717                bp->num_ethernet_queues -= diff;
1718                bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1719
1720                BNX2X_DEV_INFO("New queue configuration set: %d\n",
1721                               bp->num_queues);
1722        }
1723
1724        bp->flags |= USING_MSIX_FLAG;
1725
1726        return 0;
1727
1728no_msix:
1729        /* fall to INTx if not enough memory */
1730        if (rc == -ENOMEM)
1731                bp->flags |= DISABLE_MSI_FLAG;
1732
1733        return rc;
1734}
1735
1736static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1737{
1738        int i, rc, offset = 0;
1739
1740        /* no default status block for vf */
1741        if (IS_PF(bp)) {
1742                rc = request_irq(bp->msix_table[offset++].vector,
1743                                 bnx2x_msix_sp_int, 0,
1744                                 bp->dev->name, bp->dev);
1745                if (rc) {
1746                        BNX2X_ERR("request sp irq failed\n");
1747                        return -EBUSY;
1748                }
1749        }
1750
1751        if (CNIC_SUPPORT(bp))
1752                offset++;
1753
1754        for_each_eth_queue(bp, i) {
1755                struct bnx2x_fastpath *fp = &bp->fp[i];
1756                snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1757                         bp->dev->name, i);
1758
1759                rc = request_irq(bp->msix_table[offset].vector,
1760                                 bnx2x_msix_fp_int, 0, fp->name, fp);
1761                if (rc) {
1762                        BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1763                              bp->msix_table[offset].vector, rc);
1764                        bnx2x_free_msix_irqs(bp, offset);
1765                        return -EBUSY;
1766                }
1767
1768                offset++;
1769        }
1770
1771        i = BNX2X_NUM_ETH_QUEUES(bp);
1772        if (IS_PF(bp)) {
1773                offset = 1 + CNIC_SUPPORT(bp);
1774                netdev_info(bp->dev,
1775                            "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1776                            bp->msix_table[0].vector,
1777                            0, bp->msix_table[offset].vector,
1778                            i - 1, bp->msix_table[offset + i - 1].vector);
1779        } else {
1780                offset = CNIC_SUPPORT(bp);
1781                netdev_info(bp->dev,
1782                            "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1783                            0, bp->msix_table[offset].vector,
1784                            i - 1, bp->msix_table[offset + i - 1].vector);
1785        }
1786        return 0;
1787}
1788
1789int bnx2x_enable_msi(struct bnx2x *bp)
1790{
1791        int rc;
1792
1793        rc = pci_enable_msi(bp->pdev);
1794        if (rc) {
1795                BNX2X_DEV_INFO("MSI is not attainable\n");
1796                return -1;
1797        }
1798        bp->flags |= USING_MSI_FLAG;
1799
1800        return 0;
1801}
1802
1803static int bnx2x_req_irq(struct bnx2x *bp)
1804{
1805        unsigned long flags;
1806        unsigned int irq;
1807
1808        if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1809                flags = 0;
1810        else
1811                flags = IRQF_SHARED;
1812
1813        if (bp->flags & USING_MSIX_FLAG)
1814                irq = bp->msix_table[0].vector;
1815        else
1816                irq = bp->pdev->irq;
1817
1818        return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1819}
1820
1821static int bnx2x_setup_irqs(struct bnx2x *bp)
1822{
1823        int rc = 0;
1824        if (bp->flags & USING_MSIX_FLAG &&
1825            !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1826                rc = bnx2x_req_msix_irqs(bp);
1827                if (rc)
1828                        return rc;
1829        } else {
1830                rc = bnx2x_req_irq(bp);
1831                if (rc) {
1832                        BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1833                        return rc;
1834                }
1835                if (bp->flags & USING_MSI_FLAG) {
1836                        bp->dev->irq = bp->pdev->irq;
1837                        netdev_info(bp->dev, "using MSI IRQ %d\n",
1838                                    bp->dev->irq);
1839                }
1840                if (bp->flags & USING_MSIX_FLAG) {
1841                        bp->dev->irq = bp->msix_table[0].vector;
1842                        netdev_info(bp->dev, "using MSIX IRQ %d\n",
1843                                    bp->dev->irq);
1844                }
1845        }
1846
1847        return 0;
1848}
1849
1850static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1851{
1852        int i;
1853
1854        for_each_rx_queue_cnic(bp, i) {
1855                napi_enable(&bnx2x_fp(bp, i, napi));
1856        }
1857}
1858
1859static void bnx2x_napi_enable(struct bnx2x *bp)
1860{
1861        int i;
1862
1863        for_each_eth_queue(bp, i) {
1864                napi_enable(&bnx2x_fp(bp, i, napi));
1865        }
1866}
1867
1868static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1869{
1870        int i;
1871
1872        for_each_rx_queue_cnic(bp, i) {
1873                napi_disable(&bnx2x_fp(bp, i, napi));
1874        }
1875}
1876
1877static void bnx2x_napi_disable(struct bnx2x *bp)
1878{
1879        int i;
1880
1881        for_each_eth_queue(bp, i) {
1882                napi_disable(&bnx2x_fp(bp, i, napi));
1883        }
1884}
1885
1886void bnx2x_netif_start(struct bnx2x *bp)
1887{
1888        if (netif_running(bp->dev)) {
1889                bnx2x_napi_enable(bp);
1890                if (CNIC_LOADED(bp))
1891                        bnx2x_napi_enable_cnic(bp);
1892                bnx2x_int_enable(bp);
1893                if (bp->state == BNX2X_STATE_OPEN)
1894                        netif_tx_wake_all_queues(bp->dev);
1895        }
1896}
1897
1898void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1899{
1900        bnx2x_int_disable_sync(bp, disable_hw);
1901        bnx2x_napi_disable(bp);
1902        if (CNIC_LOADED(bp))
1903                bnx2x_napi_disable_cnic(bp);
1904}
1905
1906u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb,
1907                       void *accel_priv, select_queue_fallback_t fallback)
1908{
1909        struct bnx2x *bp = netdev_priv(dev);
1910
1911        if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1912                struct ethhdr *hdr = (struct ethhdr *)skb->data;
1913                u16 ether_type = ntohs(hdr->h_proto);
1914
1915                /* Skip VLAN tag if present */
1916                if (ether_type == ETH_P_8021Q) {
1917                        struct vlan_ethhdr *vhdr =
1918                                (struct vlan_ethhdr *)skb->data;
1919
1920                        ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1921                }
1922
1923                /* If ethertype is FCoE or FIP - use FCoE ring */
1924                if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1925                        return bnx2x_fcoe_tx(bp, txq_index);
1926        }
1927
1928        /* select a non-FCoE queue */
1929        return fallback(dev, skb) % (BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos);
1930}
1931
1932void bnx2x_set_num_queues(struct bnx2x *bp)
1933{
1934        /* RSS queues */
1935        bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1936
1937        /* override in STORAGE SD modes */
1938        if (IS_MF_STORAGE_ONLY(bp))
1939                bp->num_ethernet_queues = 1;
1940
1941        /* Add special queues */
1942        bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1943        bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1944
1945        BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1946}
1947
1948/**
1949 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1950 *
1951 * @bp:         Driver handle
1952 *
1953 * We currently support for at most 16 Tx queues for each CoS thus we will
1954 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1955 * bp->max_cos.
1956 *
1957 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1958 * index after all ETH L2 indices.
1959 *
1960 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1961 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1962 * 16..31,...) with indices that are not coupled with any real Tx queue.
1963 *
1964 * The proper configuration of skb->queue_mapping is handled by
1965 * bnx2x_select_queue() and __skb_tx_hash().
1966 *
1967 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1968 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1969 */
1970static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1971{
1972        int rc, tx, rx;
1973
1974        tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1975        rx = BNX2X_NUM_ETH_QUEUES(bp);
1976
1977/* account for fcoe queue */
1978        if (include_cnic && !NO_FCOE(bp)) {
1979                rx++;
1980                tx++;
1981        }
1982
1983        rc = netif_set_real_num_tx_queues(bp->dev, tx);
1984        if (rc) {
1985                BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1986                return rc;
1987        }
1988        rc = netif_set_real_num_rx_queues(bp->dev, rx);
1989        if (rc) {
1990                BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1991                return rc;
1992        }
1993
1994        DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1995                          tx, rx);
1996
1997        return rc;
1998}
1999
2000static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
2001{
2002        int i;
2003
2004        for_each_queue(bp, i) {
2005                struct bnx2x_fastpath *fp = &bp->fp[i];
2006                u32 mtu;
2007
2008                /* Always use a mini-jumbo MTU for the FCoE L2 ring */
2009                if (IS_FCOE_IDX(i))
2010                        /*
2011                         * Although there are no IP frames expected to arrive to
2012                         * this ring we still want to add an
2013                         * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2014                         * overrun attack.
2015                         */
2016                        mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2017                else
2018                        mtu = bp->dev->mtu;
2019                fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2020                                  IP_HEADER_ALIGNMENT_PADDING +
2021                                  ETH_OVERHEAD +
2022                                  mtu +
2023                                  BNX2X_FW_RX_ALIGN_END;
2024                fp->rx_buf_size = SKB_DATA_ALIGN(fp->rx_buf_size);
2025                /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2026                if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2027                        fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2028                else
2029                        fp->rx_frag_size = 0;
2030        }
2031}
2032
2033static int bnx2x_init_rss(struct bnx2x *bp)
2034{
2035        int i;
2036        u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2037
2038        /* Prepare the initial contents for the indirection table if RSS is
2039         * enabled
2040         */
2041        for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2042                bp->rss_conf_obj.ind_table[i] =
2043                        bp->fp->cl_id +
2044                        ethtool_rxfh_indir_default(i, num_eth_queues);
2045
2046        /*
2047         * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2048         * per-port, so if explicit configuration is needed , do it only
2049         * for a PMF.
2050         *
2051         * For 57712 and newer on the other hand it's a per-function
2052         * configuration.
2053         */
2054        return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2055}
2056
2057int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2058              bool config_hash, bool enable)
2059{
2060        struct bnx2x_config_rss_params params = {NULL};
2061
2062        /* Although RSS is meaningless when there is a single HW queue we
2063         * still need it enabled in order to have HW Rx hash generated.
2064         *
2065         * if (!is_eth_multi(bp))
2066         *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2067         */
2068
2069        params.rss_obj = rss_obj;
2070
2071        __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2072
2073        if (enable) {
2074                __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2075
2076                /* RSS configuration */
2077                __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2078                __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2079                __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2080                __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2081                if (rss_obj->udp_rss_v4)
2082                        __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2083                if (rss_obj->udp_rss_v6)
2084                        __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2085
2086                if (!CHIP_IS_E1x(bp)) {
2087                        /* valid only for TUNN_MODE_VXLAN tunnel mode */
2088                        __set_bit(BNX2X_RSS_IPV4_VXLAN, &params.rss_flags);
2089                        __set_bit(BNX2X_RSS_IPV6_VXLAN, &params.rss_flags);
2090
2091                        /* valid only for TUNN_MODE_GRE tunnel mode */
2092                        __set_bit(BNX2X_RSS_TUNN_INNER_HDRS, &params.rss_flags);
2093                }
2094        } else {
2095                __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2096        }
2097
2098        /* Hash bits */
2099        params.rss_result_mask = MULTI_MASK;
2100
2101        memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2102
2103        if (config_hash) {
2104                /* RSS keys */
2105                netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2106                __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2107        }
2108
2109        if (IS_PF(bp))
2110                return bnx2x_config_rss(bp, &params);
2111        else
2112                return bnx2x_vfpf_config_rss(bp, &params);
2113}
2114
2115static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2116{
2117        struct bnx2x_func_state_params func_params = {NULL};
2118
2119        /* Prepare parameters for function state transitions */
2120        __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2121
2122        func_params.f_obj = &bp->func_obj;
2123        func_params.cmd = BNX2X_F_CMD_HW_INIT;
2124
2125        func_params.params.hw_init.load_phase = load_code;
2126
2127        return bnx2x_func_state_change(bp, &func_params);
2128}
2129
2130/*
2131 * Cleans the object that have internal lists without sending
2132 * ramrods. Should be run when interrupts are disabled.
2133 */
2134void bnx2x_squeeze_objects(struct bnx2x *bp)
2135{
2136        int rc;
2137        unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2138        struct bnx2x_mcast_ramrod_params rparam = {NULL};
2139        struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2140
2141        /***************** Cleanup MACs' object first *************************/
2142
2143        /* Wait for completion of requested */
2144        __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2145        /* Perform a dry cleanup */
2146        __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2147
2148        /* Clean ETH primary MAC */
2149        __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2150        rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2151                                 &ramrod_flags);
2152        if (rc != 0)
2153                BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2154
2155        /* Cleanup UC list */
2156        vlan_mac_flags = 0;
2157        __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2158        rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2159                                 &ramrod_flags);
2160        if (rc != 0)
2161                BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2162
2163        /***************** Now clean mcast object *****************************/
2164        rparam.mcast_obj = &bp->mcast_obj;
2165        __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2166
2167        /* Add a DEL command... - Since we're doing a driver cleanup only,
2168         * we take a lock surrounding both the initial send and the CONTs,
2169         * as we don't want a true completion to disrupt us in the middle.
2170         */
2171        netif_addr_lock_bh(bp->dev);
2172        rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2173        if (rc < 0)
2174                BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2175                          rc);
2176
2177        /* ...and wait until all pending commands are cleared */
2178        rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2179        while (rc != 0) {
2180                if (rc < 0) {
2181                        BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2182                                  rc);
2183                        netif_addr_unlock_bh(bp->dev);
2184                        return;
2185                }
2186
2187                rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2188        }
2189        netif_addr_unlock_bh(bp->dev);
2190}
2191
2192#ifndef BNX2X_STOP_ON_ERROR
2193#define LOAD_ERROR_EXIT(bp, label) \
2194        do { \
2195                (bp)->state = BNX2X_STATE_ERROR; \
2196                goto label; \
2197        } while (0)
2198
2199#define LOAD_ERROR_EXIT_CNIC(bp, label) \
2200        do { \
2201                bp->cnic_loaded = false; \
2202                goto label; \
2203        } while (0)
2204#else /*BNX2X_STOP_ON_ERROR*/
2205#define LOAD_ERROR_EXIT(bp, label) \
2206        do { \
2207                (bp)->state = BNX2X_STATE_ERROR; \
2208                (bp)->panic = 1; \
2209                return -EBUSY; \
2210        } while (0)
2211#define LOAD_ERROR_EXIT_CNIC(bp, label) \
2212        do { \
2213                bp->cnic_loaded = false; \
2214                (bp)->panic = 1; \
2215                return -EBUSY; \
2216        } while (0)
2217#endif /*BNX2X_STOP_ON_ERROR*/
2218
2219static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2220{
2221        BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2222                       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2223        return;
2224}
2225
2226static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2227{
2228        int num_groups, vf_headroom = 0;
2229        int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2230
2231        /* number of queues for statistics is number of eth queues + FCoE */
2232        u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2233
2234        /* Total number of FW statistics requests =
2235         * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2236         * and fcoe l2 queue) stats + num of queues (which includes another 1
2237         * for fcoe l2 queue if applicable)
2238         */
2239        bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2240
2241        /* vf stats appear in the request list, but their data is allocated by
2242         * the VFs themselves. We don't include them in the bp->fw_stats_num as
2243         * it is used to determine where to place the vf stats queries in the
2244         * request struct
2245         */
2246        if (IS_SRIOV(bp))
2247                vf_headroom = bnx2x_vf_headroom(bp);
2248
2249        /* Request is built from stats_query_header and an array of
2250         * stats_query_cmd_group each of which contains
2251         * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2252         * configured in the stats_query_header.
2253         */
2254        num_groups =
2255                (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2256                 (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2257                 1 : 0));
2258
2259        DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2260           bp->fw_stats_num, vf_headroom, num_groups);
2261        bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2262                num_groups * sizeof(struct stats_query_cmd_group);
2263
2264        /* Data for statistics requests + stats_counter
2265         * stats_counter holds per-STORM counters that are incremented
2266         * when STORM has finished with the current request.
2267         * memory for FCoE offloaded statistics are counted anyway,
2268         * even if they will not be sent.
2269         * VF stats are not accounted for here as the data of VF stats is stored
2270         * in memory allocated by the VF, not here.
2271         */
2272        bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2273                sizeof(struct per_pf_stats) +
2274                sizeof(struct fcoe_statistics_params) +
2275                sizeof(struct per_queue_stats) * num_queue_stats +
2276                sizeof(struct stats_counter);
2277
2278        bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2279                                       bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2280        if (!bp->fw_stats)
2281                goto alloc_mem_err;
2282
2283        /* Set shortcuts */
2284        bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2285        bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2286        bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2287                ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2288        bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2289                bp->fw_stats_req_sz;
2290
2291        DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2292           U64_HI(bp->fw_stats_req_mapping),
2293           U64_LO(bp->fw_stats_req_mapping));
2294        DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2295           U64_HI(bp->fw_stats_data_mapping),
2296           U64_LO(bp->fw_stats_data_mapping));
2297        return 0;
2298
2299alloc_mem_err:
2300        bnx2x_free_fw_stats_mem(bp);
2301        BNX2X_ERR("Can't allocate FW stats memory\n");
2302        return -ENOMEM;
2303}
2304
2305/* send load request to mcp and analyze response */
2306static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2307{
2308        u32 param;
2309
2310        /* init fw_seq */
2311        bp->fw_seq =
2312                (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2313                 DRV_MSG_SEQ_NUMBER_MASK);
2314        BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2315
2316        /* Get current FW pulse sequence */
2317        bp->fw_drv_pulse_wr_seq =
2318                (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2319                 DRV_PULSE_SEQ_MASK);
2320        BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2321
2322        param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2323
2324        if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2325                param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2326
2327        /* load request */
2328        (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2329
2330        /* if mcp fails to respond we must abort */
2331        if (!(*load_code)) {
2332                BNX2X_ERR("MCP response failure, aborting\n");
2333                return -EBUSY;
2334        }
2335
2336        /* If mcp refused (e.g. other port is in diagnostic mode) we
2337         * must abort
2338         */
2339        if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2340                BNX2X_ERR("MCP refused load request, aborting\n");
2341                return -EBUSY;
2342        }
2343        return 0;
2344}
2345
2346/* check whether another PF has already loaded FW to chip. In
2347 * virtualized environments a pf from another VM may have already
2348 * initialized the device including loading FW
2349 */
2350int bnx2x_compare_fw_ver(struct bnx2x *bp, u32 load_code, bool print_err)
2351{
2352        /* is another pf loaded on this engine? */
2353        if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2354            load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2355                /* build my FW version dword */
2356                u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2357                        (BCM_5710_FW_MINOR_VERSION << 8) +
2358                        (BCM_5710_FW_REVISION_VERSION << 16) +
2359                        (BCM_5710_FW_ENGINEERING_VERSION << 24);
2360
2361                /* read loaded FW from chip */
2362                u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2363
2364                DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2365                   loaded_fw, my_fw);
2366
2367                /* abort nic load if version mismatch */
2368                if (my_fw != loaded_fw) {
2369                        if (print_err)
2370                                BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2371                                          loaded_fw, my_fw);
2372                        else
2373                                BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2374                                               loaded_fw, my_fw);
2375                        return -EBUSY;
2376                }
2377        }
2378        return 0;
2379}
2380
2381/* returns the "mcp load_code" according to global load_count array */
2382static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2383{
2384        int path = BP_PATH(bp);
2385
2386        DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2387           path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2388           bnx2x_load_count[path][2]);
2389        bnx2x_load_count[path][0]++;
2390        bnx2x_load_count[path][1 + port]++;
2391        DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2392           path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2393           bnx2x_load_count[path][2]);
2394        if (bnx2x_load_count[path][0] == 1)
2395                return FW_MSG_CODE_DRV_LOAD_COMMON;
2396        else if (bnx2x_load_count[path][1 + port] == 1)
2397                return FW_MSG_CODE_DRV_LOAD_PORT;
2398        else
2399                return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2400}
2401
2402/* mark PMF if applicable */
2403static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2404{
2405        if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2406            (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2407            (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2408                bp->port.pmf = 1;
2409                /* We need the barrier to ensure the ordering between the
2410                 * writing to bp->port.pmf here and reading it from the
2411                 * bnx2x_periodic_task().
2412                 */
2413                smp_mb();
2414        } else {
2415                bp->port.pmf = 0;
2416        }
2417
2418        DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2419}
2420
2421static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2422{
2423        if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2424             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2425            (bp->common.shmem2_base)) {
2426                if (SHMEM2_HAS(bp, dcc_support))
2427                        SHMEM2_WR(bp, dcc_support,
2428                                  (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2429                                   SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2430                if (SHMEM2_HAS(bp, afex_driver_support))
2431                        SHMEM2_WR(bp, afex_driver_support,
2432                                  SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2433        }
2434
2435        /* Set AFEX default VLAN tag to an invalid value */
2436        bp->afex_def_vlan_tag = -1;
2437}
2438
2439/**
2440 * bnx2x_bz_fp - zero content of the fastpath structure.
2441 *
2442 * @bp:         driver handle
2443 * @index:      fastpath index to be zeroed
2444 *
2445 * Makes sure the contents of the bp->fp[index].napi is kept
2446 * intact.
2447 */
2448static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2449{
2450        struct bnx2x_fastpath *fp = &bp->fp[index];
2451        int cos;
2452        struct napi_struct orig_napi = fp->napi;
2453        struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2454
2455        /* bzero bnx2x_fastpath contents */
2456        if (fp->tpa_info)
2457                memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2458                       sizeof(struct bnx2x_agg_info));
2459        memset(fp, 0, sizeof(*fp));
2460
2461        /* Restore the NAPI object as it has been already initialized */
2462        fp->napi = orig_napi;
2463        fp->tpa_info = orig_tpa_info;
2464        fp->bp = bp;
2465        fp->index = index;
2466        if (IS_ETH_FP(fp))
2467                fp->max_cos = bp->max_cos;
2468        else
2469                /* Special queues support only one CoS */
2470                fp->max_cos = 1;
2471
2472        /* Init txdata pointers */
2473        if (IS_FCOE_FP(fp))
2474                fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2475        if (IS_ETH_FP(fp))
2476                for_each_cos_in_tx_queue(fp, cos)
2477                        fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2478                                BNX2X_NUM_ETH_QUEUES(bp) + index];
2479
2480        /* set the tpa flag for each queue. The tpa flag determines the queue
2481         * minimal size so it must be set prior to queue memory allocation
2482         */
2483        if (bp->dev->features & NETIF_F_LRO)
2484                fp->mode = TPA_MODE_LRO;
2485        else if (bp->dev->features & NETIF_F_GRO &&
2486                 bnx2x_mtu_allows_gro(bp->dev->mtu))
2487                fp->mode = TPA_MODE_GRO;
2488        else
2489                fp->mode = TPA_MODE_DISABLED;
2490
2491        /* We don't want TPA if it's disabled in bp
2492         * or if this is an FCoE L2 ring.
2493         */
2494        if (bp->disable_tpa || IS_FCOE_FP(fp))
2495                fp->mode = TPA_MODE_DISABLED;
2496}
2497
2498void bnx2x_set_os_driver_state(struct bnx2x *bp, u32 state)
2499{
2500        u32 cur;
2501
2502        if (!IS_MF_BD(bp) || !SHMEM2_HAS(bp, os_driver_state) || IS_VF(bp))
2503                return;
2504
2505        cur = SHMEM2_RD(bp, os_driver_state[BP_FW_MB_IDX(bp)]);
2506        DP(NETIF_MSG_IFUP, "Driver state %08x-->%08x\n",
2507           cur, state);
2508
2509        SHMEM2_WR(bp, os_driver_state[BP_FW_MB_IDX(bp)], state);
2510}
2511
2512int bnx2x_load_cnic(struct bnx2x *bp)
2513{
2514        int i, rc, port = BP_PORT(bp);
2515
2516        DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2517
2518        mutex_init(&bp->cnic_mutex);
2519
2520        if (IS_PF(bp)) {
2521                rc = bnx2x_alloc_mem_cnic(bp);
2522                if (rc) {
2523                        BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2524                        LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2525                }
2526        }
2527
2528        rc = bnx2x_alloc_fp_mem_cnic(bp);
2529        if (rc) {
2530                BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2531                LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2532        }
2533
2534        /* Update the number of queues with the cnic queues */
2535        rc = bnx2x_set_real_num_queues(bp, 1);
2536        if (rc) {
2537                BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2538                LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2539        }
2540
2541        /* Add all CNIC NAPI objects */
2542        bnx2x_add_all_napi_cnic(bp);
2543        DP(NETIF_MSG_IFUP, "cnic napi added\n");
2544        bnx2x_napi_enable_cnic(bp);
2545
2546        rc = bnx2x_init_hw_func_cnic(bp);
2547        if (rc)
2548                LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2549
2550        bnx2x_nic_init_cnic(bp);
2551
2552        if (IS_PF(bp)) {
2553                /* Enable Timer scan */
2554                REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2555
2556                /* setup cnic queues */
2557                for_each_cnic_queue(bp, i) {
2558                        rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2559                        if (rc) {
2560                                BNX2X_ERR("Queue setup failed\n");
2561                                LOAD_ERROR_EXIT(bp, load_error_cnic2);
2562                        }
2563                }
2564        }
2565
2566        /* Initialize Rx filter. */
2567        bnx2x_set_rx_mode_inner(bp);
2568
2569        /* re-read iscsi info */
2570        bnx2x_get_iscsi_info(bp);
2571        bnx2x_setup_cnic_irq_info(bp);
2572        bnx2x_setup_cnic_info(bp);
2573        bp->cnic_loaded = true;
2574        if (bp->state == BNX2X_STATE_OPEN)
2575                bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2576
2577        DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2578
2579        return 0;
2580
2581#ifndef BNX2X_STOP_ON_ERROR
2582load_error_cnic2:
2583        /* Disable Timer scan */
2584        REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2585
2586load_error_cnic1:
2587        bnx2x_napi_disable_cnic(bp);
2588        /* Update the number of queues without the cnic queues */
2589        if (bnx2x_set_real_num_queues(bp, 0))
2590                BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2591load_error_cnic0:
2592        BNX2X_ERR("CNIC-related load failed\n");
2593        bnx2x_free_fp_mem_cnic(bp);
2594        bnx2x_free_mem_cnic(bp);
2595        return rc;
2596#endif /* ! BNX2X_STOP_ON_ERROR */
2597}
2598
2599/* must be called with rtnl_lock */
2600int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2601{
2602        int port = BP_PORT(bp);
2603        int i, rc = 0, load_code = 0;
2604
2605        DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2606        DP(NETIF_MSG_IFUP,
2607           "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2608
2609#ifdef BNX2X_STOP_ON_ERROR
2610        if (unlikely(bp->panic)) {
2611                BNX2X_ERR("Can't load NIC when there is panic\n");
2612                return -EPERM;
2613        }
2614#endif
2615
2616        bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2617
2618        /* zero the structure w/o any lock, before SP handler is initialized */
2619        memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2620        __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2621                &bp->last_reported_link.link_report_flags);
2622
2623        if (IS_PF(bp))
2624                /* must be called before memory allocation and HW init */
2625                bnx2x_ilt_set_info(bp);
2626
2627        /*
2628         * Zero fastpath structures preserving invariants like napi, which are
2629         * allocated only once, fp index, max_cos, bp pointer.
2630         * Also set fp->mode and txdata_ptr.
2631         */
2632        DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2633        for_each_queue(bp, i)
2634                bnx2x_bz_fp(bp, i);
2635        memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2636                                  bp->num_cnic_queues) *
2637                                  sizeof(struct bnx2x_fp_txdata));
2638
2639        bp->fcoe_init = false;
2640
2641        /* Set the receive queues buffer size */
2642        bnx2x_set_rx_buf_size(bp);
2643
2644        if (IS_PF(bp)) {
2645                rc = bnx2x_alloc_mem(bp);
2646                if (rc) {
2647                        BNX2X_ERR("Unable to allocate bp memory\n");
2648                        return rc;
2649                }
2650        }
2651
2652        /* need to be done after alloc mem, since it's self adjusting to amount
2653         * of memory available for RSS queues
2654         */
2655        rc = bnx2x_alloc_fp_mem(bp);
2656        if (rc) {
2657                BNX2X_ERR("Unable to allocate memory for fps\n");
2658                LOAD_ERROR_EXIT(bp, load_error0);
2659        }
2660
2661        /* Allocated memory for FW statistics  */
2662        if (bnx2x_alloc_fw_stats_mem(bp))
2663                LOAD_ERROR_EXIT(bp, load_error0);
2664
2665        /* request pf to initialize status blocks */
2666        if (IS_VF(bp)) {
2667                rc = bnx2x_vfpf_init(bp);
2668                if (rc)
2669                        LOAD_ERROR_EXIT(bp, load_error0);
2670        }
2671
2672        /* As long as bnx2x_alloc_mem() may possibly update
2673         * bp->num_queues, bnx2x_set_real_num_queues() should always
2674         * come after it. At this stage cnic queues are not counted.
2675         */
2676        rc = bnx2x_set_real_num_queues(bp, 0);
2677        if (rc) {
2678                BNX2X_ERR("Unable to set real_num_queues\n");
2679                LOAD_ERROR_EXIT(bp, load_error0);
2680        }
2681
2682        /* configure multi cos mappings in kernel.
2683         * this configuration may be overridden by a multi class queue
2684         * discipline or by a dcbx negotiation result.
2685         */
2686        bnx2x_setup_tc(bp->dev, bp->max_cos);
2687
2688        /* Add all NAPI objects */
2689        bnx2x_add_all_napi(bp);
2690        DP(NETIF_MSG_IFUP, "napi added\n");
2691        bnx2x_napi_enable(bp);
2692
2693        if (IS_PF(bp)) {
2694                /* set pf load just before approaching the MCP */
2695                bnx2x_set_pf_load(bp);
2696
2697                /* if mcp exists send load request and analyze response */
2698                if (!BP_NOMCP(bp)) {
2699                        /* attempt to load pf */
2700                        rc = bnx2x_nic_load_request(bp, &load_code);
2701                        if (rc)
2702                                LOAD_ERROR_EXIT(bp, load_error1);
2703
2704                        /* what did mcp say? */
2705                        rc = bnx2x_compare_fw_ver(bp, load_code, true);
2706                        if (rc) {
2707                                bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2708                                LOAD_ERROR_EXIT(bp, load_error2);
2709                        }
2710                } else {
2711                        load_code = bnx2x_nic_load_no_mcp(bp, port);
2712                }
2713
2714                /* mark pmf if applicable */
2715                bnx2x_nic_load_pmf(bp, load_code);
2716
2717                /* Init Function state controlling object */
2718                bnx2x__init_func_obj(bp);
2719
2720                /* Initialize HW */
2721                rc = bnx2x_init_hw(bp, load_code);
2722                if (rc) {
2723                        BNX2X_ERR("HW init failed, aborting\n");
2724                        bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2725                        LOAD_ERROR_EXIT(bp, load_error2);
2726                }
2727        }
2728
2729        bnx2x_pre_irq_nic_init(bp);
2730
2731        /* Connect to IRQs */
2732        rc = bnx2x_setup_irqs(bp);
2733        if (rc) {
2734                BNX2X_ERR("setup irqs failed\n");
2735                if (IS_PF(bp))
2736                        bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2737                LOAD_ERROR_EXIT(bp, load_error2);
2738        }
2739
2740        /* Init per-function objects */
2741        if (IS_PF(bp)) {
2742                /* Setup NIC internals and enable interrupts */
2743                bnx2x_post_irq_nic_init(bp, load_code);
2744
2745                bnx2x_init_bp_objs(bp);
2746                bnx2x_iov_nic_init(bp);
2747
2748                /* Set AFEX default VLAN tag to an invalid value */
2749                bp->afex_def_vlan_tag = -1;
2750                bnx2x_nic_load_afex_dcc(bp, load_code);
2751                bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2752                rc = bnx2x_func_start(bp);
2753                if (rc) {
2754                        BNX2X_ERR("Function start failed!\n");
2755                        bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2756
2757                        LOAD_ERROR_EXIT(bp, load_error3);
2758                }
2759
2760                /* Send LOAD_DONE command to MCP */
2761                if (!BP_NOMCP(bp)) {
2762                        load_code = bnx2x_fw_command(bp,
2763                                                     DRV_MSG_CODE_LOAD_DONE, 0);
2764                        if (!load_code) {
2765                                BNX2X_ERR("MCP response failure, aborting\n");
2766                                rc = -EBUSY;
2767                                LOAD_ERROR_EXIT(bp, load_error3);
2768                        }
2769                }
2770
2771                /* initialize FW coalescing state machines in RAM */
2772                bnx2x_update_coalesce(bp);
2773        }
2774
2775        /* setup the leading queue */
2776        rc = bnx2x_setup_leading(bp);
2777        if (rc) {
2778                BNX2X_ERR("Setup leading failed!\n");
2779                LOAD_ERROR_EXIT(bp, load_error3);
2780        }
2781
2782        /* set up the rest of the queues */
2783        for_each_nondefault_eth_queue(bp, i) {
2784                if (IS_PF(bp))
2785                        rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2786                else /* VF */
2787                        rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2788                if (rc) {
2789                        BNX2X_ERR("Queue %d setup failed\n", i);
2790                        LOAD_ERROR_EXIT(bp, load_error3);
2791                }
2792        }
2793
2794        /* setup rss */
2795        rc = bnx2x_init_rss(bp);
2796        if (rc) {
2797                BNX2X_ERR("PF RSS init failed\n");
2798                LOAD_ERROR_EXIT(bp, load_error3);
2799        }
2800
2801        /* Now when Clients are configured we are ready to work */
2802        bp->state = BNX2X_STATE_OPEN;
2803
2804        /* Configure a ucast MAC */
2805        if (IS_PF(bp))
2806                rc = bnx2x_set_eth_mac(bp, true);
2807        else /* vf */
2808                rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2809                                           true);
2810        if (rc) {
2811                BNX2X_ERR("Setting Ethernet MAC failed\n");
2812                LOAD_ERROR_EXIT(bp, load_error3);
2813        }
2814
2815        if (IS_PF(bp) && bp->pending_max) {
2816                bnx2x_update_max_mf_config(bp, bp->pending_max);
2817                bp->pending_max = 0;
2818        }
2819
2820        if (bp->port.pmf) {
2821                rc = bnx2x_initial_phy_init(bp, load_mode);
2822                if (rc)
2823                        LOAD_ERROR_EXIT(bp, load_error3);
2824        }
2825        bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2826
2827        /* Start fast path */
2828
2829        /* Re-configure vlan filters */
2830        rc = bnx2x_vlan_reconfigure_vid(bp);
2831        if (rc)
2832                LOAD_ERROR_EXIT(bp, load_error3);
2833
2834        /* Initialize Rx filter. */
2835        bnx2x_set_rx_mode_inner(bp);
2836
2837        if (bp->flags & PTP_SUPPORTED) {
2838                bnx2x_init_ptp(bp);
2839                bnx2x_configure_ptp_filters(bp);
2840        }
2841        /* Start Tx */
2842        switch (load_mode) {
2843        case LOAD_NORMAL:
2844                /* Tx queue should be only re-enabled */
2845                netif_tx_wake_all_queues(bp->dev);
2846                break;
2847
2848        case LOAD_OPEN:
2849                netif_tx_start_all_queues(bp->dev);
2850                smp_mb__after_atomic();
2851                break;
2852
2853        case LOAD_DIAG:
2854        case LOAD_LOOPBACK_EXT:
2855                bp->state = BNX2X_STATE_DIAG;
2856                break;
2857
2858        default:
2859                break;
2860        }
2861
2862        if (bp->port.pmf)
2863                bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2864        else
2865                bnx2x__link_status_update(bp);
2866
2867        /* start the timer */
2868        mod_timer(&bp->timer, jiffies + bp->current_interval);
2869
2870        if (CNIC_ENABLED(bp))
2871                bnx2x_load_cnic(bp);
2872
2873        if (IS_PF(bp))
2874                bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2875
2876        if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2877                /* mark driver is loaded in shmem2 */
2878                u32 val;
2879                val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2880                val &= ~DRV_FLAGS_MTU_MASK;
2881                val |= (bp->dev->mtu << DRV_FLAGS_MTU_SHIFT);
2882                SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2883                          val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2884                          DRV_FLAGS_CAPABILITIES_LOADED_L2);
2885        }
2886
2887        /* Wait for all pending SP commands to complete */
2888        if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2889                BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2890                bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2891                return -EBUSY;
2892        }
2893
2894        /* Update driver data for On-Chip MFW dump. */
2895        if (IS_PF(bp))
2896                bnx2x_update_mfw_dump(bp);
2897
2898        /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2899        if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2900                bnx2x_dcbx_init(bp, false);
2901
2902        if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2903                bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_ACTIVE);
2904
2905        DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2906
2907        return 0;
2908
2909#ifndef BNX2X_STOP_ON_ERROR
2910load_error3:
2911        if (IS_PF(bp)) {
2912                bnx2x_int_disable_sync(bp, 1);
2913
2914                /* Clean queueable objects */
2915                bnx2x_squeeze_objects(bp);
2916        }
2917
2918        /* Free SKBs, SGEs, TPA pool and driver internals */
2919        bnx2x_free_skbs(bp);
2920        for_each_rx_queue(bp, i)
2921                bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2922
2923        /* Release IRQs */
2924        bnx2x_free_irq(bp);
2925load_error2:
2926        if (IS_PF(bp) && !BP_NOMCP(bp)) {
2927                bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2928                bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2929        }
2930
2931        bp->port.pmf = 0;
2932load_error1:
2933        bnx2x_napi_disable(bp);
2934        bnx2x_del_all_napi(bp);
2935
2936        /* clear pf_load status, as it was already set */
2937        if (IS_PF(bp))
2938                bnx2x_clear_pf_load(bp);
2939load_error0:
2940        bnx2x_free_fw_stats_mem(bp);
2941        bnx2x_free_fp_mem(bp);
2942        bnx2x_free_mem(bp);
2943
2944        return rc;
2945#endif /* ! BNX2X_STOP_ON_ERROR */
2946}
2947
2948int bnx2x_drain_tx_queues(struct bnx2x *bp)
2949{
2950        u8 rc = 0, cos, i;
2951
2952        /* Wait until tx fastpath tasks complete */
2953        for_each_tx_queue(bp, i) {
2954                struct bnx2x_fastpath *fp = &bp->fp[i];
2955
2956                for_each_cos_in_tx_queue(fp, cos)
2957                        rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2958                if (rc)
2959                        return rc;
2960        }
2961        return 0;
2962}
2963
2964/* must be called with rtnl_lock */
2965int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2966{
2967        int i;
2968        bool global = false;
2969
2970        DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2971
2972        if (!IS_MF_SD_STORAGE_PERSONALITY_ONLY(bp))
2973                bnx2x_set_os_driver_state(bp, OS_DRIVER_STATE_DISABLED);
2974
2975        /* mark driver is unloaded in shmem2 */
2976        if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2977                u32 val;
2978                val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2979                SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2980                          val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2981        }
2982
2983        if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2984            (bp->state == BNX2X_STATE_CLOSED ||
2985             bp->state == BNX2X_STATE_ERROR)) {
2986                /* We can get here if the driver has been unloaded
2987                 * during parity error recovery and is either waiting for a
2988                 * leader to complete or for other functions to unload and
2989                 * then ifdown has been issued. In this case we want to
2990                 * unload and let other functions to complete a recovery
2991                 * process.
2992                 */
2993                bp->recovery_state = BNX2X_RECOVERY_DONE;
2994                bp->is_leader = 0;
2995                bnx2x_release_leader_lock(bp);
2996                smp_mb();
2997
2998                DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2999                BNX2X_ERR("Can't unload in closed or error state\n");
3000                return -EINVAL;
3001        }
3002
3003        /* Nothing to do during unload if previous bnx2x_nic_load()
3004         * have not completed successfully - all resources are released.
3005         *
3006         * we can get here only after unsuccessful ndo_* callback, during which
3007         * dev->IFF_UP flag is still on.
3008         */
3009        if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
3010                return 0;
3011
3012        /* It's important to set the bp->state to the value different from
3013         * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
3014         * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
3015         */
3016        bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
3017        smp_mb();
3018
3019        /* indicate to VFs that the PF is going down */
3020        bnx2x_iov_channel_down(bp);
3021
3022        if (CNIC_LOADED(bp))
3023                bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
3024
3025        /* Stop Tx */
3026        bnx2x_tx_disable(bp);
3027        netdev_reset_tc(bp->dev);
3028
3029        bp->rx_mode = BNX2X_RX_MODE_NONE;
3030
3031        del_timer_sync(&bp->timer);
3032
3033        if (IS_PF(bp)) {
3034                /* Set ALWAYS_ALIVE bit in shmem */
3035                bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
3036                bnx2x_drv_pulse(bp);
3037                bnx2x_stats_handle(bp, STATS_EVENT_STOP);
3038                bnx2x_save_statistics(bp);
3039        }
3040
3041        /* wait till consumers catch up with producers in all queues.
3042         * If we're recovering, FW can't write to host so no reason
3043         * to wait for the queues to complete all Tx.
3044         */
3045        if (unload_mode != UNLOAD_RECOVERY)
3046                bnx2x_drain_tx_queues(bp);
3047
3048        /* if VF indicate to PF this function is going down (PF will delete sp
3049         * elements and clear initializations
3050         */
3051        if (IS_VF(bp))
3052                bnx2x_vfpf_close_vf(bp);
3053        else if (unload_mode != UNLOAD_RECOVERY)
3054                /* if this is a normal/close unload need to clean up chip*/
3055                bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3056        else {
3057                /* Send the UNLOAD_REQUEST to the MCP */
3058                bnx2x_send_unload_req(bp, unload_mode);
3059
3060                /* Prevent transactions to host from the functions on the
3061                 * engine that doesn't reset global blocks in case of global
3062                 * attention once global blocks are reset and gates are opened
3063                 * (the engine which leader will perform the recovery
3064                 * last).
3065                 */
3066                if (!CHIP_IS_E1x(bp))
3067                        bnx2x_pf_disable(bp);
3068
3069                /* Disable HW interrupts, NAPI */
3070                bnx2x_netif_stop(bp, 1);
3071                /* Delete all NAPI objects */
3072                bnx2x_del_all_napi(bp);
3073                if (CNIC_LOADED(bp))
3074                        bnx2x_del_all_napi_cnic(bp);
3075                /* Release IRQs */
3076                bnx2x_free_irq(bp);
3077
3078                /* Report UNLOAD_DONE to MCP */
3079                bnx2x_send_unload_done(bp, false);
3080        }
3081
3082        /*
3083         * At this stage no more interrupts will arrive so we may safely clean
3084         * the queueable objects here in case they failed to get cleaned so far.
3085         */
3086        if (IS_PF(bp))
3087                bnx2x_squeeze_objects(bp);
3088
3089        /* There should be no more pending SP commands at this stage */
3090        bp->sp_state = 0;
3091
3092        bp->port.pmf = 0;
3093
3094        /* clear pending work in rtnl task */
3095        bp->sp_rtnl_state = 0;
3096        smp_mb();
3097
3098        /* Free SKBs, SGEs, TPA pool and driver internals */
3099        bnx2x_free_skbs(bp);
3100        if (CNIC_LOADED(bp))
3101                bnx2x_free_skbs_cnic(bp);
3102        for_each_rx_queue(bp, i)
3103                bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3104
3105        bnx2x_free_fp_mem(bp);
3106        if (CNIC_LOADED(bp))
3107                bnx2x_free_fp_mem_cnic(bp);
3108
3109        if (IS_PF(bp)) {
3110                if (CNIC_LOADED(bp))
3111                        bnx2x_free_mem_cnic(bp);
3112        }
3113        bnx2x_free_mem(bp);
3114
3115        bp->state = BNX2X_STATE_CLOSED;
3116        bp->cnic_loaded = false;
3117
3118        /* Clear driver version indication in shmem */
3119        if (IS_PF(bp))
3120                bnx2x_update_mng_version(bp);
3121
3122        /* Check if there are pending parity attentions. If there are - set
3123         * RECOVERY_IN_PROGRESS.
3124         */
3125        if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3126                bnx2x_set_reset_in_progress(bp);
3127
3128                /* Set RESET_IS_GLOBAL if needed */
3129                if (global)
3130                        bnx2x_set_reset_global(bp);
3131        }
3132
3133        /* The last driver must disable a "close the gate" if there is no
3134         * parity attention or "process kill" pending.
3135         */
3136        if (IS_PF(bp) &&
3137            !bnx2x_clear_pf_load(bp) &&
3138            bnx2x_reset_is_done(bp, BP_PATH(bp)))
3139                bnx2x_disable_close_the_gate(bp);
3140
3141        DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3142
3143        return 0;
3144}
3145
3146int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3147{
3148        u16 pmcsr;
3149
3150        /* If there is no power capability, silently succeed */
3151        if (!bp->pdev->pm_cap) {
3152                BNX2X_DEV_INFO("No power capability. Breaking.\n");
3153                return 0;
3154        }
3155
3156        pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3157
3158        switch (state) {
3159        case PCI_D0:
3160                pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3161                                      ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3162                                       PCI_PM_CTRL_PME_STATUS));
3163
3164                if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3165                        /* delay required during transition out of D3hot */
3166                        msleep(20);
3167                break;
3168
3169        case PCI_D3hot:
3170                /* If there are other clients above don't
3171                   shut down the power */
3172                if (atomic_read(&bp->pdev->enable_cnt) != 1)
3173                        return 0;
3174                /* Don't shut down the power for emulation and FPGA */
3175                if (CHIP_REV_IS_SLOW(bp))
3176                        return 0;
3177
3178                pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3179                pmcsr |= 3;
3180
3181                if (bp->wol)
3182                        pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3183
3184                pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3185                                      pmcsr);
3186
3187                /* No more memory access after this point until
3188                * device is brought back to D0.
3189                */
3190                break;
3191
3192        default:
3193                dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3194                return -EINVAL;
3195        }
3196        return 0;
3197}
3198
3199/*
3200 * net_device service functions
3201 */
3202static int bnx2x_poll(struct napi_struct *napi, int budget)
3203{
3204        struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3205                                                 napi);
3206        struct bnx2x *bp = fp->bp;
3207        int rx_work_done;
3208        u8 cos;
3209
3210#ifdef BNX2X_STOP_ON_ERROR
3211        if (unlikely(bp->panic)) {
3212                napi_complete(napi);
3213                return 0;
3214        }
3215#endif
3216        for_each_cos_in_tx_queue(fp, cos)
3217                if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3218                        bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3219
3220        rx_work_done = (bnx2x_has_rx_work(fp)) ? bnx2x_rx_int(fp, budget) : 0;
3221
3222        if (rx_work_done < budget) {
3223                /* No need to update SB for FCoE L2 ring as long as
3224                 * it's connected to the default SB and the SB
3225                 * has been updated when NAPI was scheduled.
3226                 */
3227                if (IS_FCOE_FP(fp)) {
3228                        napi_complete_done(napi, rx_work_done);
3229                } else {
3230                        bnx2x_update_fpsb_idx(fp);
3231                        /* bnx2x_has_rx_work() reads the status block,
3232                         * thus we need to ensure that status block indices
3233                         * have been actually read (bnx2x_update_fpsb_idx)
3234                         * prior to this check (bnx2x_has_rx_work) so that
3235                         * we won't write the "newer" value of the status block
3236                         * to IGU (if there was a DMA right after
3237                         * bnx2x_has_rx_work and if there is no rmb, the memory
3238                         * reading (bnx2x_update_fpsb_idx) may be postponed
3239                         * to right before bnx2x_ack_sb). In this case there
3240                         * will never be another interrupt until there is
3241                         * another update of the status block, while there
3242                         * is still unhandled work.
3243                         */
3244                        rmb();
3245
3246                        if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3247                                if (napi_complete_done(napi, rx_work_done)) {
3248                                        /* Re-enable interrupts */
3249                                        DP(NETIF_MSG_RX_STATUS,
3250                                           "Update index to %d\n", fp->fp_hc_idx);
3251                                        bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3252                                                     le16_to_cpu(fp->fp_hc_idx),
3253                                                     IGU_INT_ENABLE, 1);
3254                                }
3255                        } else {
3256                                rx_work_done = budget;
3257                        }
3258                }
3259        }
3260
3261        return rx_work_done;
3262}
3263
3264/* we split the first BD into headers and data BDs
3265 * to ease the pain of our fellow microcode engineers
3266 * we use one mapping for both BDs
3267 */
3268static u16 bnx2x_tx_split(struct bnx2x *bp,
3269                          struct bnx2x_fp_txdata *txdata,
3270                          struct sw_tx_bd *tx_buf,
3271                          struct eth_tx_start_bd **tx_bd, u16 hlen,
3272                          u16 bd_prod)
3273{
3274        struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3275        struct eth_tx_bd *d_tx_bd;
3276        dma_addr_t mapping;
3277        int old_len = le16_to_cpu(h_tx_bd->nbytes);
3278
3279        /* first fix first BD */
3280        h_tx_bd->nbytes = cpu_to_le16(hlen);
3281
3282        DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3283           h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3284
3285        /* now get a new data BD
3286         * (after the pbd) and fill it */
3287        bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3288        d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3289
3290        mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3291                           le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3292
3293        d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3294        d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3295        d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3296
3297        /* this marks the BD as one that has no individual mapping */
3298        tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3299
3300        DP(NETIF_MSG_TX_QUEUED,
3301           "TSO split data size is %d (%x:%x)\n",
3302           d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3303
3304        /* update tx_bd */
3305        *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3306
3307        return bd_prod;
3308}
3309
3310#define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3311#define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3312static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3313{
3314        __sum16 tsum = (__force __sum16) csum;
3315
3316        if (fix > 0)
3317                tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3318                                  csum_partial(t_header - fix, fix, 0)));
3319
3320        else if (fix < 0)
3321                tsum = ~csum_fold(csum_add((__force __wsum) csum,
3322                                  csum_partial(t_header, -fix, 0)));
3323
3324        return bswab16(tsum);
3325}
3326
3327static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3328{
3329        u32 rc;
3330        __u8 prot = 0;
3331        __be16 protocol;
3332
3333        if (skb->ip_summed != CHECKSUM_PARTIAL)
3334                return XMIT_PLAIN;
3335
3336        protocol = vlan_get_protocol(skb);
3337        if (protocol == htons(ETH_P_IPV6)) {
3338                rc = XMIT_CSUM_V6;
3339                prot = ipv6_hdr(skb)->nexthdr;
3340        } else {
3341                rc = XMIT_CSUM_V4;
3342                prot = ip_hdr(skb)->protocol;
3343        }
3344
3345        if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3346                if (inner_ip_hdr(skb)->version == 6) {
3347                        rc |= XMIT_CSUM_ENC_V6;
3348                        if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3349                                rc |= XMIT_CSUM_TCP;
3350                } else {
3351                        rc |= XMIT_CSUM_ENC_V4;
3352                        if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3353                                rc |= XMIT_CSUM_TCP;
3354                }
3355        }
3356        if (prot == IPPROTO_TCP)
3357                rc |= XMIT_CSUM_TCP;
3358
3359        if (skb_is_gso(skb)) {
3360                if (skb_is_gso_v6(skb)) {
3361                        rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3362                        if (rc & XMIT_CSUM_ENC)
3363                                rc |= XMIT_GSO_ENC_V6;
3364                } else {
3365                        rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3366                        if (rc & XMIT_CSUM_ENC)
3367                                rc |= XMIT_GSO_ENC_V4;
3368                }
3369        }
3370
3371        return rc;
3372}
3373
3374/* VXLAN: 4 = 1 (for linear data BD) + 3 (2 for PBD and last BD) */
3375#define BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS         4
3376
3377/* Regular: 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3378#define BNX2X_NUM_TSO_WIN_SUB_BDS               3
3379
3380#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3381/* check if packet requires linearization (packet is too fragmented)
3382   no need to check fragmentation if page size > 8K (there will be no
3383   violation to FW restrictions) */
3384static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3385                             u32 xmit_type)
3386{
3387        int first_bd_sz = 0, num_tso_win_sub = BNX2X_NUM_TSO_WIN_SUB_BDS;
3388        int to_copy = 0, hlen = 0;
3389
3390        if (xmit_type & XMIT_GSO_ENC)
3391                num_tso_win_sub = BNX2X_NUM_VXLAN_TSO_WIN_SUB_BDS;
3392
3393        if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - num_tso_win_sub)) {
3394                if (xmit_type & XMIT_GSO) {
3395                        unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3396                        int wnd_size = MAX_FETCH_BD - num_tso_win_sub;
3397                        /* Number of windows to check */
3398                        int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3399                        int wnd_idx = 0;
3400                        int frag_idx = 0;
3401                        u32 wnd_sum = 0;
3402
3403                        /* Headers length */
3404                        if (xmit_type & XMIT_GSO_ENC)
3405                                hlen = (int)(skb_inner_transport_header(skb) -
3406                                             skb->data) +
3407                                             inner_tcp_hdrlen(skb);
3408                        else
3409                                hlen = (int)(skb_transport_header(skb) -
3410                                             skb->data) + tcp_hdrlen(skb);
3411
3412                        /* Amount of data (w/o headers) on linear part of SKB*/
3413                        first_bd_sz = skb_headlen(skb) - hlen;
3414
3415                        wnd_sum  = first_bd_sz;
3416
3417                        /* Calculate the first sum - it's special */
3418                        for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3419                                wnd_sum +=
3420                                        skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3421
3422                        /* If there was data on linear skb data - check it */
3423                        if (first_bd_sz > 0) {
3424                                if (unlikely(wnd_sum < lso_mss)) {
3425                                        to_copy = 1;
3426                                        goto exit_lbl;
3427                                }
3428
3429                                wnd_sum -= first_bd_sz;
3430                        }
3431
3432                        /* Others are easier: run through the frag list and
3433                           check all windows */
3434                        for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3435                                wnd_sum +=
3436                          skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3437
3438                                if (unlikely(wnd_sum < lso_mss)) {
3439                                        to_copy = 1;
3440                                        break;
3441                                }
3442                                wnd_sum -=
3443                                        skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3444                        }
3445                } else {
3446                        /* in non-LSO too fragmented packet should always
3447                           be linearized */
3448                        to_copy = 1;
3449                }
3450        }
3451
3452exit_lbl:
3453        if (unlikely(to_copy))
3454                DP(NETIF_MSG_TX_QUEUED,
3455                   "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3456                   (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3457                   skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3458
3459        return to_copy;
3460}
3461#endif
3462
3463/**
3464 * bnx2x_set_pbd_gso - update PBD in GSO case.
3465 *
3466 * @skb:        packet skb
3467 * @pbd:        parse BD
3468 * @xmit_type:  xmit flags
3469 */
3470static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3471                              struct eth_tx_parse_bd_e1x *pbd,
3472                              u32 xmit_type)
3473{
3474        pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3475        pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3476        pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3477
3478        if (xmit_type & XMIT_GSO_V4) {
3479                pbd->ip_id = bswab16(ip_hdr(skb)->id);
3480                pbd->tcp_pseudo_csum =
3481                        bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3482                                                   ip_hdr(skb)->daddr,
3483                                                   0, IPPROTO_TCP, 0));
3484        } else {
3485                pbd->tcp_pseudo_csum =
3486                        bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3487                                                 &ipv6_hdr(skb)->daddr,
3488                                                 0, IPPROTO_TCP, 0));
3489        }
3490
3491        pbd->global_data |=
3492                cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3493}
3494
3495/**
3496 * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3497 *
3498 * @bp:                 driver handle
3499 * @skb:                packet skb
3500 * @parsing_data:       data to be updated
3501 * @xmit_type:          xmit flags
3502 *
3503 * 57712/578xx related, when skb has encapsulation
3504 */
3505static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3506                                 u32 *parsing_data, u32 xmit_type)
3507{
3508        *parsing_data |=
3509                ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3510                ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3511                ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3512
3513        if (xmit_type & XMIT_CSUM_TCP) {
3514                *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3515                        ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3516                        ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3517
3518                return skb_inner_transport_header(skb) +
3519                        inner_tcp_hdrlen(skb) - skb->data;
3520        }
3521
3522        /* We support checksum offload for TCP and UDP only.
3523         * No need to pass the UDP header length - it's a constant.
3524         */
3525        return skb_inner_transport_header(skb) +
3526                sizeof(struct udphdr) - skb->data;
3527}
3528
3529/**
3530 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3531 *
3532 * @bp:                 driver handle
3533 * @skb:                packet skb
3534 * @parsing_data:       data to be updated
3535 * @xmit_type:          xmit flags
3536 *
3537 * 57712/578xx related
3538 */
3539static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3540                                u32 *parsing_data, u32 xmit_type)
3541{
3542        *parsing_data |=
3543                ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3544                ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3545                ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3546
3547        if (xmit_type & XMIT_CSUM_TCP) {
3548                *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3549                        ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3550                        ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3551
3552                return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3553        }
3554        /* We support checksum offload for TCP and UDP only.
3555         * No need to pass the UDP header length - it's a constant.
3556         */
3557        return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3558}
3559
3560/* set FW indication according to inner or outer protocols if tunneled */
3561static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3562                               struct eth_tx_start_bd *tx_start_bd,
3563                               u32 xmit_type)
3564{
3565        tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3566
3567        if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3568                tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3569
3570        if (!(xmit_type & XMIT_CSUM_TCP))
3571                tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3572}
3573
3574/**
3575 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3576 *
3577 * @bp:         driver handle
3578 * @skb:        packet skb
3579 * @pbd:        parse BD to be updated
3580 * @xmit_type:  xmit flags
3581 */
3582static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3583                             struct eth_tx_parse_bd_e1x *pbd,
3584                             u32 xmit_type)
3585{
3586        u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3587
3588        /* for now NS flag is not used in Linux */
3589        pbd->global_data =
3590                cpu_to_le16(hlen |
3591                            ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3592                             ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3593
3594        pbd->ip_hlen_w = (skb_transport_header(skb) -
3595                        skb_network_header(skb)) >> 1;
3596
3597        hlen += pbd->ip_hlen_w;
3598
3599        /* We support checksum offload for TCP and UDP only */
3600        if (xmit_type & XMIT_CSUM_TCP)
3601                hlen += tcp_hdrlen(skb) / 2;
3602        else
3603                hlen += sizeof(struct udphdr) / 2;
3604
3605        pbd->total_hlen_w = cpu_to_le16(hlen);
3606        hlen = hlen*2;
3607
3608        if (xmit_type & XMIT_CSUM_TCP) {
3609                pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3610
3611        } else {
3612                s8 fix = SKB_CS_OFF(skb); /* signed! */
3613
3614                DP(NETIF_MSG_TX_QUEUED,
3615                   "hlen %d  fix %d  csum before fix %x\n",
3616                   le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3617
3618                /* HW bug: fixup the CSUM */
3619                pbd->tcp_pseudo_csum =
3620                        bnx2x_csum_fix(skb_transport_header(skb),
3621                                       SKB_CS(skb), fix);
3622
3623                DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3624                   pbd->tcp_pseudo_csum);
3625        }
3626
3627        return hlen;
3628}
3629
3630static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3631                                      struct eth_tx_parse_bd_e2 *pbd_e2,
3632                                      struct eth_tx_parse_2nd_bd *pbd2,
3633                                      u16 *global_data,
3634                                      u32 xmit_type)
3635{
3636        u16 hlen_w = 0;
3637        u8 outerip_off, outerip_len = 0;
3638
3639        /* from outer IP to transport */
3640        hlen_w = (skb_inner_transport_header(skb) -
3641                  skb_network_header(skb)) >> 1;
3642
3643        /* transport len */
3644        hlen_w += inner_tcp_hdrlen(skb) >> 1;
3645
3646        pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3647
3648        /* outer IP header info */
3649        if (xmit_type & XMIT_CSUM_V4) {
3650                struct iphdr *iph = ip_hdr(skb);
3651                u32 csum = (__force u32)(~iph->check) -
3652                           (__force u32)iph->tot_len -
3653                           (__force u32)iph->frag_off;
3654
3655                outerip_len = iph->ihl << 1;
3656
3657                pbd2->fw_ip_csum_wo_len_flags_frag =
3658                        bswab16(csum_fold((__force __wsum)csum));
3659        } else {
3660                pbd2->fw_ip_hdr_to_payload_w =
3661                        hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3662                pbd_e2->data.tunnel_data.flags |=
3663                        ETH_TUNNEL_DATA_IPV6_OUTER;
3664        }
3665
3666        pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3667
3668        pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3669
3670        /* inner IP header info */
3671        if (xmit_type & XMIT_CSUM_ENC_V4) {
3672                pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3673
3674                pbd_e2->data.tunnel_data.pseudo_csum =
3675                        bswab16(~csum_tcpudp_magic(
3676                                        inner_ip_hdr(skb)->saddr,
3677                                        inner_ip_hdr(skb)->daddr,
3678                                        0, IPPROTO_TCP, 0));
3679        } else {
3680                pbd_e2->data.tunnel_data.pseudo_csum =
3681                        bswab16(~csum_ipv6_magic(
3682                                        &inner_ipv6_hdr(skb)->saddr,
3683                                        &inner_ipv6_hdr(skb)->daddr,
3684                                        0, IPPROTO_TCP, 0));
3685        }
3686
3687        outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3688
3689        *global_data |=
3690                outerip_off |
3691                (outerip_len <<
3692                        ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3693                ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3694                        ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3695
3696        if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3697                SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3698                pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3699        }
3700}
3701
3702static inline void bnx2x_set_ipv6_ext_e2(struct sk_buff *skb, u32 *parsing_data,
3703                                         u32 xmit_type)
3704{
3705        struct ipv6hdr *ipv6;
3706
3707        if (!(xmit_type & (XMIT_GSO_ENC_V6 | XMIT_GSO_V6)))
3708                return;
3709
3710        if (xmit_type & XMIT_GSO_ENC_V6)
3711                ipv6 = inner_ipv6_hdr(skb);
3712        else /* XMIT_GSO_V6 */
3713                ipv6 = ipv6_hdr(skb);
3714
3715        if (ipv6->nexthdr == NEXTHDR_IPV6)
3716                *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3717}
3718
3719/* called with netif_tx_lock
3720 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3721 * netif_wake_queue()
3722 */
3723netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3724{
3725        struct bnx2x *bp = netdev_priv(dev);
3726
3727        struct netdev_queue *txq;
3728        struct bnx2x_fp_txdata *txdata;
3729        struct sw_tx_bd *tx_buf;
3730        struct eth_tx_start_bd *tx_start_bd, *first_bd;
3731        struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3732        struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3733        struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3734        struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3735        u32 pbd_e2_parsing_data = 0;
3736        u16 pkt_prod, bd_prod;
3737        int nbd, txq_index;
3738        dma_addr_t mapping;
3739        u32 xmit_type = bnx2x_xmit_type(bp, skb);
3740        int i;
3741        u8 hlen = 0;
3742        __le16 pkt_size = 0;
3743        struct ethhdr *eth;
3744        u8 mac_type = UNICAST_ADDRESS;
3745
3746#ifdef BNX2X_STOP_ON_ERROR
3747        if (unlikely(bp->panic))
3748                return NETDEV_TX_BUSY;
3749#endif
3750
3751        txq_index = skb_get_queue_mapping(skb);
3752        txq = netdev_get_tx_queue(dev, txq_index);
3753
3754        BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3755
3756        txdata = &bp->bnx2x_txq[txq_index];
3757
3758        /* enable this debug print to view the transmission queue being used
3759        DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3760           txq_index, fp_index, txdata_index); */
3761
3762        /* enable this debug print to view the transmission details
3763        DP(NETIF_MSG_TX_QUEUED,
3764           "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3765           txdata->cid, fp_index, txdata_index, txdata, fp); */
3766
3767        if (unlikely(bnx2x_tx_avail(bp, txdata) <
3768                        skb_shinfo(skb)->nr_frags +
3769                        BDS_PER_TX_PKT +
3770                        NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3771                /* Handle special storage cases separately */
3772                if (txdata->tx_ring_size == 0) {
3773                        struct bnx2x_eth_q_stats *q_stats =
3774                                bnx2x_fp_qstats(bp, txdata->parent_fp);
3775                        q_stats->driver_filtered_tx_pkt++;
3776                        dev_kfree_skb(skb);
3777                        return NETDEV_TX_OK;
3778                }
3779                bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3780                netif_tx_stop_queue(txq);
3781                BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3782
3783                return NETDEV_TX_BUSY;
3784        }
3785
3786        DP(NETIF_MSG_TX_QUEUED,
3787           "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3788           txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3789           ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3790           skb->len);
3791
3792        eth = (struct ethhdr *)skb->data;
3793
3794        /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3795        if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3796                if (is_broadcast_ether_addr(eth->h_dest))
3797                        mac_type = BROADCAST_ADDRESS;
3798                else
3799                        mac_type = MULTICAST_ADDRESS;
3800        }
3801
3802#if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3803        /* First, check if we need to linearize the skb (due to FW
3804           restrictions). No need to check fragmentation if page size > 8K
3805           (there will be no violation to FW restrictions) */
3806        if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3807                /* Statistics of linearization */
3808                bp->lin_cnt++;
3809                if (skb_linearize(skb) != 0) {
3810                        DP(NETIF_MSG_TX_QUEUED,
3811                           "SKB linearization failed - silently dropping this SKB\n");
3812                        dev_kfree_skb_any(skb);
3813                        return NETDEV_TX_OK;
3814                }
3815        }
3816#endif
3817        /* Map skb linear data for DMA */
3818        mapping = dma_map_single(&bp->pdev->dev, skb->data,
3819                                 skb_headlen(skb), DMA_TO_DEVICE);
3820        if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3821                DP(NETIF_MSG_TX_QUEUED,
3822                   "SKB mapping failed - silently dropping this SKB\n");
3823                dev_kfree_skb_any(skb);
3824                return NETDEV_TX_OK;
3825        }
3826        /*
3827        Please read carefully. First we use one BD which we mark as start,
3828        then we have a parsing info BD (used for TSO or xsum),
3829        and only then we have the rest of the TSO BDs.
3830        (don't forget to mark the last one as last,
3831        and to unmap only AFTER you write to the BD ...)
3832        And above all, all pdb sizes are in words - NOT DWORDS!
3833        */
3834
3835        /* get current pkt produced now - advance it just before sending packet
3836         * since mapping of pages may fail and cause packet to be dropped
3837         */
3838        pkt_prod = txdata->tx_pkt_prod;
3839        bd_prod = TX_BD(txdata->tx_bd_prod);
3840
3841        /* get a tx_buf and first BD
3842         * tx_start_bd may be changed during SPLIT,
3843         * but first_bd will always stay first
3844         */
3845        tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3846        tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3847        first_bd = tx_start_bd;
3848
3849        tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3850
3851        if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
3852                if (!(bp->flags & TX_TIMESTAMPING_EN)) {
3853                        BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n");
3854                } else if (bp->ptp_tx_skb) {
3855                        BNX2X_ERR("The device supports only a single outstanding packet to timestamp, this packet will not be timestamped\n");
3856                } else {
3857                        skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3858                        /* schedule check for Tx timestamp */
3859                        bp->ptp_tx_skb = skb_get(skb);
3860                        bp->ptp_tx_start = jiffies;
3861                        schedule_work(&bp->ptp_task);
3862                }
3863        }
3864
3865        /* header nbd: indirectly zero other flags! */
3866        tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3867
3868        /* remember the first BD of the packet */
3869        tx_buf->first_bd = txdata->tx_bd_prod;
3870        tx_buf->skb = skb;
3871        tx_buf->flags = 0;
3872
3873        DP(NETIF_MSG_TX_QUEUED,
3874           "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3875           pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3876
3877        if (skb_vlan_tag_present(skb)) {
3878                tx_start_bd->vlan_or_ethertype =
3879                    cpu_to_le16(skb_vlan_tag_get(skb));
3880                tx_start_bd->bd_flags.as_bitfield |=
3881                    (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3882        } else {
3883                /* when transmitting in a vf, start bd must hold the ethertype
3884                 * for fw to enforce it
3885                 */
3886                u16 vlan_tci = 0;
3887#ifndef BNX2X_STOP_ON_ERROR
3888                if (IS_VF(bp)) {
3889#endif
3890                        /* Still need to consider inband vlan for enforced */
3891                        if (__vlan_get_tag(skb, &vlan_tci)) {
3892                                tx_start_bd->vlan_or_ethertype =
3893                                        cpu_to_le16(ntohs(eth->h_proto));
3894                        } else {
3895                                tx_start_bd->bd_flags.as_bitfield |=
3896                                        (X_ETH_INBAND_VLAN <<
3897                                         ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3898                                tx_start_bd->vlan_or_ethertype =
3899                                        cpu_to_le16(vlan_tci);
3900                        }
3901#ifndef BNX2X_STOP_ON_ERROR
3902                } else {
3903                        /* used by FW for packet accounting */
3904                        tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3905                }
3906#endif
3907        }
3908
3909        nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3910
3911        /* turn on parsing and get a BD */
3912        bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3913
3914        if (xmit_type & XMIT_CSUM)
3915                bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3916
3917        if (!CHIP_IS_E1x(bp)) {
3918                pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3919                memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3920
3921                if (xmit_type & XMIT_CSUM_ENC) {
3922                        u16 global_data = 0;
3923
3924                        /* Set PBD in enc checksum offload case */
3925                        hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3926                                                      &pbd_e2_parsing_data,
3927                                                      xmit_type);
3928
3929                        /* turn on 2nd parsing and get a BD */
3930                        bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3931
3932                        pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3933
3934                        memset(pbd2, 0, sizeof(*pbd2));
3935
3936                        pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3937                                (skb_inner_network_header(skb) -
3938                                 skb->data) >> 1;
3939
3940                        if (xmit_type & XMIT_GSO_ENC)
3941                                bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3942                                                          &global_data,
3943                                                          xmit_type);
3944
3945                        pbd2->global_data = cpu_to_le16(global_data);
3946
3947                        /* add addition parse BD indication to start BD */
3948                        SET_FLAG(tx_start_bd->general_data,
3949                                 ETH_TX_START_BD_PARSE_NBDS, 1);
3950                        /* set encapsulation flag in start BD */
3951                        SET_FLAG(tx_start_bd->general_data,
3952                                 ETH_TX_START_BD_TUNNEL_EXIST, 1);
3953
3954                        tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3955
3956                        nbd++;
3957                } else if (xmit_type & XMIT_CSUM) {
3958                        /* Set PBD in checksum offload case w/o encapsulation */
3959                        hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3960                                                     &pbd_e2_parsing_data,
3961                                                     xmit_type);
3962                }
3963
3964                bnx2x_set_ipv6_ext_e2(skb, &pbd_e2_parsing_data, xmit_type);
3965                /* Add the macs to the parsing BD if this is a vf or if
3966                 * Tx Switching is enabled.
3967                 */
3968                if (IS_VF(bp)) {
3969                        /* override GRE parameters in BD */
3970                        bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3971                                              &pbd_e2->data.mac_addr.src_mid,
3972                                              &pbd_e2->data.mac_addr.src_lo,
3973                                              eth->h_source);
3974
3975                        bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3976                                              &pbd_e2->data.mac_addr.dst_mid,
3977                                              &pbd_e2->data.mac_addr.dst_lo,
3978                                              eth->h_dest);
3979                } else {
3980                        if (bp->flags & TX_SWITCHING)
3981                                bnx2x_set_fw_mac_addr(
3982                                                &pbd_e2->data.mac_addr.dst_hi,
3983                                                &pbd_e2->data.mac_addr.dst_mid,
3984                                                &pbd_e2->data.mac_addr.dst_lo,
3985                                                eth->h_dest);
3986#ifdef BNX2X_STOP_ON_ERROR
3987                        /* Enforce security is always set in Stop on Error -
3988                         * source mac should be present in the parsing BD
3989                         */
3990                        bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3991                                              &pbd_e2->data.mac_addr.src_mid,
3992                                              &pbd_e2->data.mac_addr.src_lo,
3993                                              eth->h_source);
3994#endif
3995                }
3996
3997                SET_FLAG(pbd_e2_parsing_data,
3998                         ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3999        } else {
4000                u16 global_data = 0;
4001                pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
4002                memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
4003                /* Set PBD in checksum offload case */
4004                if (xmit_type & XMIT_CSUM)
4005                        hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
4006
4007                SET_FLAG(global_data,
4008                         ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
4009                pbd_e1x->global_data |= cpu_to_le16(global_data);
4010        }
4011
4012        /* Setup the data pointer of the first BD of the packet */
4013        tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4014        tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4015        tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
4016        pkt_size = tx_start_bd->nbytes;
4017
4018        DP(NETIF_MSG_TX_QUEUED,
4019           "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
4020           tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
4021           le16_to_cpu(tx_start_bd->nbytes),
4022           tx_start_bd->bd_flags.as_bitfield,
4023           le16_to_cpu(tx_start_bd->vlan_or_ethertype));
4024
4025        if (xmit_type & XMIT_GSO) {
4026
4027                DP(NETIF_MSG_TX_QUEUED,
4028                   "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
4029                   skb->len, hlen, skb_headlen(skb),
4030                   skb_shinfo(skb)->gso_size);
4031
4032                tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
4033
4034                if (unlikely(skb_headlen(skb) > hlen)) {
4035                        nbd++;
4036                        bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
4037                                                 &tx_start_bd, hlen,
4038                                                 bd_prod);
4039                }
4040                if (!CHIP_IS_E1x(bp))
4041                        pbd_e2_parsing_data |=
4042                                (skb_shinfo(skb)->gso_size <<
4043                                 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
4044                                 ETH_TX_PARSE_BD_E2_LSO_MSS;
4045                else
4046                        bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
4047        }
4048
4049        /* Set the PBD's parsing_data field if not zero
4050         * (for the chips newer than 57711).
4051         */
4052        if (pbd_e2_parsing_data)
4053                pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
4054
4055        tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
4056
4057        /* Handle fragmented skb */
4058        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4059                skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4060
4061                mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
4062                                           skb_frag_size(frag), DMA_TO_DEVICE);
4063                if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
4064                        unsigned int pkts_compl = 0, bytes_compl = 0;
4065
4066                        DP(NETIF_MSG_TX_QUEUED,
4067                           "Unable to map page - dropping packet...\n");
4068
4069                        /* we need unmap all buffers already mapped
4070                         * for this SKB;
4071                         * first_bd->nbd need to be properly updated
4072                         * before call to bnx2x_free_tx_pkt
4073                         */
4074                        first_bd->nbd = cpu_to_le16(nbd);
4075                        bnx2x_free_tx_pkt(bp, txdata,
4076                                          TX_BD(txdata->tx_pkt_prod),
4077                                          &pkts_compl, &bytes_compl);
4078                        return NETDEV_TX_OK;
4079                }
4080
4081                bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4082                tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4083                if (total_pkt_bd == NULL)
4084                        total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
4085
4086                tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
4087                tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
4088                tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
4089                le16_add_cpu(&pkt_size, skb_frag_size(frag));
4090                nbd++;
4091
4092                DP(NETIF_MSG_TX_QUEUED,
4093                   "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
4094                   i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
4095                   le16_to_cpu(tx_data_bd->nbytes));
4096        }
4097
4098        DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
4099
4100        /* update with actual num BDs */
4101        first_bd->nbd = cpu_to_le16(nbd);
4102
4103        bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
4104
4105        /* now send a tx doorbell, counting the next BD
4106         * if the packet contains or ends with it
4107         */
4108        if (TX_BD_POFF(bd_prod) < nbd)
4109                nbd++;
4110
4111        /* total_pkt_bytes should be set on the first data BD if
4112         * it's not an LSO packet and there is more than one
4113         * data BD. In this case pkt_size is limited by an MTU value.
4114         * However we prefer to set it for an LSO packet (while we don't
4115         * have to) in order to save some CPU cycles in a none-LSO
4116         * case, when we much more care about them.
4117         */
4118        if (total_pkt_bd != NULL)
4119                total_pkt_bd->total_pkt_bytes = pkt_size;
4120
4121        if (pbd_e1x)
4122                DP(NETIF_MSG_TX_QUEUED,
4123                   "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
4124                   pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
4125                   pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
4126                   pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
4127                    le16_to_cpu(pbd_e1x->total_hlen_w));
4128        if (pbd_e2)
4129                DP(NETIF_MSG_TX_QUEUED,
4130                   "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
4131                   pbd_e2,
4132                   pbd_e2->data.mac_addr.dst_hi,
4133                   pbd_e2->data.mac_addr.dst_mid,
4134                   pbd_e2->data.mac_addr.dst_lo,
4135                   pbd_e2->data.mac_addr.src_hi,
4136                   pbd_e2->data.mac_addr.src_mid,
4137                   pbd_e2->data.mac_addr.src_lo,
4138                   pbd_e2->parsing_data);
4139        DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
4140
4141        netdev_tx_sent_queue(txq, skb->len);
4142
4143        skb_tx_timestamp(skb);
4144
4145        txdata->tx_pkt_prod++;
4146        /*
4147         * Make sure that the BD data is updated before updating the producer
4148         * since FW might read the BD right after the producer is updated.
4149         * This is only applicable for weak-ordered memory model archs such
4150         * as IA-64. The following barrier is also mandatory since FW will
4151         * assumes packets must have BDs.
4152         */
4153        wmb();
4154
4155        txdata->tx_db.data.prod += nbd;
4156        barrier();
4157
4158        DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4159
4160        mmiowb();
4161
4162        txdata->tx_bd_prod += nbd;
4163
4164        if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4165                netif_tx_stop_queue(txq);
4166
4167                /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4168                 * ordering of set_bit() in netif_tx_stop_queue() and read of
4169                 * fp->bd_tx_cons */
4170                smp_mb();
4171
4172                bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4173                if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4174                        netif_tx_wake_queue(txq);
4175        }
4176        txdata->tx_pkt++;
4177
4178        return NETDEV_TX_OK;
4179}
4180
4181void bnx2x_get_c2s_mapping(struct bnx2x *bp, u8 *c2s_map, u8 *c2s_default)
4182{
4183        int mfw_vn = BP_FW_MB_IDX(bp);
4184        u32 tmp;
4185
4186        /* If the shmem shouldn't affect configuration, reflect */
4187        if (!IS_MF_BD(bp)) {
4188                int i;
4189
4190                for (i = 0; i < BNX2X_MAX_PRIORITY; i++)
4191                        c2s_map[i] = i;
4192                *c2s_default = 0;
4193
4194                return;
4195        }
4196
4197        tmp = SHMEM2_RD(bp, c2s_pcp_map_lower[mfw_vn]);
4198        tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4199        c2s_map[0] = tmp & 0xff;
4200        c2s_map[1] = (tmp >> 8) & 0xff;
4201        c2s_map[2] = (tmp >> 16) & 0xff;
4202        c2s_map[3] = (tmp >> 24) & 0xff;
4203
4204        tmp = SHMEM2_RD(bp, c2s_pcp_map_upper[mfw_vn]);
4205        tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4206        c2s_map[4] = tmp & 0xff;
4207        c2s_map[5] = (tmp >> 8) & 0xff;
4208        c2s_map[6] = (tmp >> 16) & 0xff;
4209        c2s_map[7] = (tmp >> 24) & 0xff;
4210
4211        tmp = SHMEM2_RD(bp, c2s_pcp_map_default[mfw_vn]);
4212        tmp = (__force u32)be32_to_cpu((__force __be32)tmp);
4213        *c2s_default = (tmp >> (8 * mfw_vn)) & 0xff;
4214}
4215
4216/**
4217 * bnx2x_setup_tc - routine to configure net_device for multi tc
4218 *
4219 * @netdev: net device to configure
4220 * @tc: number of traffic classes to enable
4221 *
4222 * callback connected to the ndo_setup_tc function pointer
4223 */
4224int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4225{
4226        struct bnx2x *bp = netdev_priv(dev);
4227        u8 c2s_map[BNX2X_MAX_PRIORITY], c2s_def;
4228        int cos, prio, count, offset;
4229
4230        /* setup tc must be called under rtnl lock */
4231        ASSERT_RTNL();
4232
4233        /* no traffic classes requested. Aborting */
4234        if (!num_tc) {
4235                netdev_reset_tc(dev);
4236                return 0;
4237        }
4238
4239        /* requested to support too many traffic classes */
4240        if (num_tc > bp->max_cos) {
4241                BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4242                          num_tc, bp->max_cos);
4243                return -EINVAL;
4244        }
4245
4246        /* declare amount of supported traffic classes */
4247        if (netdev_set_num_tc(dev, num_tc)) {
4248                BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4249                return -EINVAL;
4250        }
4251
4252        bnx2x_get_c2s_mapping(bp, c2s_map, &c2s_def);
4253
4254        /* configure priority to traffic class mapping */
4255        for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4256                int outer_prio = c2s_map[prio];
4257
4258                netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[outer_prio]);
4259                DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4260                   "mapping priority %d to tc %d\n",
4261                   outer_prio, bp->prio_to_cos[outer_prio]);
4262        }
4263
4264        /* Use this configuration to differentiate tc0 from other COSes
4265           This can be used for ets or pfc, and save the effort of setting
4266           up a multio class queue disc or negotiating DCBX with a switch
4267        netdev_set_prio_tc_map(dev, 0, 0);
4268        DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4269        for (prio = 1; prio < 16; prio++) {
4270                netdev_set_prio_tc_map(dev, prio, 1);
4271                DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4272        } */
4273
4274        /* configure traffic class to transmission queue mapping */
4275        for (cos = 0; cos < bp->max_cos; cos++) {
4276                count = BNX2X_NUM_ETH_QUEUES(bp);
4277                offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4278                netdev_set_tc_queue(dev, cos, count, offset);
4279                DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4280                   "mapping tc %d to offset %d count %d\n",
4281                   cos, offset, count);
4282        }
4283
4284        return 0;
4285}
4286
4287int __bnx2x_setup_tc(struct net_device *dev, enum tc_setup_type type,
4288                     void *type_data)
4289{
4290        struct tc_mqprio_qopt *mqprio = type_data;
4291
4292        if (type != TC_SETUP_MQPRIO)
4293                return -EOPNOTSUPP;
4294
4295        mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
4296
4297        return bnx2x_setup_tc(dev, mqprio->num_tc);
4298}
4299
4300/* called with rtnl_lock */
4301int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4302{
4303        struct sockaddr *addr = p;
4304        struct bnx2x *bp = netdev_priv(dev);
4305        int rc = 0;
4306
4307        if (!is_valid_ether_addr(addr->sa_data)) {
4308                BNX2X_ERR("Requested MAC address is not valid\n");
4309                return -EINVAL;
4310        }
4311
4312        if (IS_MF_STORAGE_ONLY(bp)) {
4313                BNX2X_ERR("Can't change address on STORAGE ONLY function\n");
4314                return -EINVAL;
4315        }
4316
4317        if (netif_running(dev))  {
4318                rc = bnx2x_set_eth_mac(bp, false);
4319                if (rc)
4320                        return rc;
4321        }
4322
4323        memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4324
4325        if (netif_running(dev))
4326                rc = bnx2x_set_eth_mac(bp, true);
4327
4328        if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4329                SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4330
4331        return rc;
4332}
4333
4334static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4335{
4336        union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4337        struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4338        u8 cos;
4339
4340        /* Common */
4341
4342        if (IS_FCOE_IDX(fp_index)) {
4343                memset(sb, 0, sizeof(union host_hc_status_block));
4344                fp->status_blk_mapping = 0;
4345        } else {
4346                /* status blocks */
4347                if (!CHIP_IS_E1x(bp))
4348                        BNX2X_PCI_FREE(sb->e2_sb,
4349                                       bnx2x_fp(bp, fp_index,
4350                                                status_blk_mapping),
4351                                       sizeof(struct host_hc_status_block_e2));
4352                else
4353                        BNX2X_PCI_FREE(sb->e1x_sb,
4354                                       bnx2x_fp(bp, fp_index,
4355                                                status_blk_mapping),
4356                                       sizeof(struct host_hc_status_block_e1x));
4357        }
4358
4359        /* Rx */
4360        if (!skip_rx_queue(bp, fp_index)) {
4361                bnx2x_free_rx_bds(fp);
4362
4363                /* fastpath rx rings: rx_buf rx_desc rx_comp */
4364                BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4365                BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4366                               bnx2x_fp(bp, fp_index, rx_desc_mapping),
4367                               sizeof(struct eth_rx_bd) * NUM_RX_BD);
4368
4369                BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4370                               bnx2x_fp(bp, fp_index, rx_comp_mapping),
4371                               sizeof(struct eth_fast_path_rx_cqe) *
4372                               NUM_RCQ_BD);
4373
4374                /* SGE ring */
4375                BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4376                BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4377                               bnx2x_fp(bp, fp_index, rx_sge_mapping),
4378                               BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4379        }
4380
4381        /* Tx */
4382        if (!skip_tx_queue(bp, fp_index)) {
4383                /* fastpath tx rings: tx_buf tx_desc */
4384                for_each_cos_in_tx_queue(fp, cos) {
4385                        struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4386
4387                        DP(NETIF_MSG_IFDOWN,
4388                           "freeing tx memory of fp %d cos %d cid %d\n",
4389                           fp_index, cos, txdata->cid);
4390
4391                        BNX2X_FREE(txdata->tx_buf_ring);
4392                        BNX2X_PCI_FREE(txdata->tx_desc_ring,
4393                                txdata->tx_desc_mapping,
4394                                sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4395                }
4396        }
4397        /* end of fastpath */
4398}
4399
4400static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4401{
4402        int i;
4403        for_each_cnic_queue(bp, i)
4404                bnx2x_free_fp_mem_at(bp, i);
4405}
4406
4407void bnx2x_free_fp_mem(struct bnx2x *bp)
4408{
4409        int i;
4410        for_each_eth_queue(bp, i)
4411                bnx2x_free_fp_mem_at(bp, i);
4412}
4413
4414static void set_sb_shortcuts(struct bnx2x *bp, int index)
4415{
4416        union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4417        if (!CHIP_IS_E1x(bp)) {
4418                bnx2x_fp(bp, index, sb_index_values) =
4419                        (__le16 *)status_blk.e2_sb->sb.index_values;
4420                bnx2x_fp(bp, index, sb_running_index) =
4421                        (__le16 *)status_blk.e2_sb->sb.running_index;
4422        } else {
4423                bnx2x_fp(bp, index, sb_index_values) =
4424                        (__le16 *)status_blk.e1x_sb->sb.index_values;
4425                bnx2x_fp(bp, index, sb_running_index) =
4426                        (__le16 *)status_blk.e1x_sb->sb.running_index;
4427        }
4428}
4429
4430/* Returns the number of actually allocated BDs */
4431static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4432                              int rx_ring_size)
4433{
4434        struct bnx2x *bp = fp->bp;
4435        u16 ring_prod, cqe_ring_prod;
4436        int i, failure_cnt = 0;
4437
4438        fp->rx_comp_cons = 0;
4439        cqe_ring_prod = ring_prod = 0;
4440
4441        /* This routine is called only during fo init so
4442         * fp->eth_q_stats.rx_skb_alloc_failed = 0
4443         */
4444        for (i = 0; i < rx_ring_size; i++) {
4445                if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4446                        failure_cnt++;
4447                        continue;
4448                }
4449                ring_prod = NEXT_RX_IDX(ring_prod);
4450                cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4451                WARN_ON(ring_prod <= (i - failure_cnt));
4452        }
4453
4454        if (failure_cnt)
4455                BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4456                          i - failure_cnt, fp->index);
4457
4458        fp->rx_bd_prod = ring_prod;
4459        /* Limit the CQE producer by the CQE ring size */
4460        fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4461                               cqe_ring_prod);
4462
4463        bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4464
4465        return i - failure_cnt;
4466}
4467
4468static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4469{
4470        int i;
4471
4472        for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4473                struct eth_rx_cqe_next_page *nextpg;
4474
4475                nextpg = (struct eth_rx_cqe_next_page *)
4476                        &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4477                nextpg->addr_hi =
4478                        cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4479                                   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4480                nextpg->addr_lo =
4481                        cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4482                                   BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4483        }
4484}
4485
4486static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4487{
4488        union host_hc_status_block *sb;
4489        struct bnx2x_fastpath *fp = &bp->fp[index];
4490        int ring_size = 0;
4491        u8 cos;
4492        int rx_ring_size = 0;
4493
4494        if (!bp->rx_ring_size && IS_MF_STORAGE_ONLY(bp)) {
4495                rx_ring_size = MIN_RX_SIZE_NONTPA;
4496                bp->rx_ring_size = rx_ring_size;
4497        } else if (!bp->rx_ring_size) {
4498                rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4499
4500                if (CHIP_IS_E3(bp)) {
4501                        u32 cfg = SHMEM_RD(bp,
4502                                           dev_info.port_hw_config[BP_PORT(bp)].
4503                                           default_cfg);
4504
4505                        /* Decrease ring size for 1G functions */
4506                        if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4507                            PORT_HW_CFG_NET_SERDES_IF_SGMII)
4508                                rx_ring_size /= 10;
4509                }
4510
4511                /* allocate at least number of buffers required by FW */
4512                rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4513                                     MIN_RX_SIZE_TPA, rx_ring_size);
4514
4515                bp->rx_ring_size = rx_ring_size;
4516        } else /* if rx_ring_size specified - use it */
4517                rx_ring_size = bp->rx_ring_size;
4518
4519        DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4520
4521        /* Common */
4522        sb = &bnx2x_fp(bp, index, status_blk);
4523
4524        if (!IS_FCOE_IDX(index)) {
4525                /* status blocks */
4526                if (!CHIP_IS_E1x(bp)) {
4527                        sb->e2_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4528                                                    sizeof(struct host_hc_status_block_e2));
4529                        if (!sb->e2_sb)
4530                                goto alloc_mem_err;
4531                } else {
4532                        sb->e1x_sb = BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, status_blk_mapping),
4533                                                     sizeof(struct host_hc_status_block_e1x));
4534                        if (!sb->e1x_sb)
4535                                goto alloc_mem_err;
4536                }
4537        }
4538
4539        /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4540         * set shortcuts for it.
4541         */
4542        if (!IS_FCOE_IDX(index))
4543                set_sb_shortcuts(bp, index);
4544
4545        /* Tx */
4546        if (!skip_tx_queue(bp, index)) {
4547                /* fastpath tx rings: tx_buf tx_desc */
4548                for_each_cos_in_tx_queue(fp, cos) {
4549                        struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4550
4551                        DP(NETIF_MSG_IFUP,
4552                           "allocating tx memory of fp %d cos %d\n",
4553                           index, cos);
4554
4555                        txdata->tx_buf_ring = kcalloc(NUM_TX_BD,
4556                                                      sizeof(struct sw_tx_bd),
4557                                                      GFP_KERNEL);
4558                        if (!txdata->tx_buf_ring)
4559                                goto alloc_mem_err;
4560                        txdata->tx_desc_ring = BNX2X_PCI_ALLOC(&txdata->tx_desc_mapping,
4561                                                               sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4562                        if (!txdata->tx_desc_ring)
4563                                goto alloc_mem_err;
4564                }
4565        }
4566
4567        /* Rx */
4568        if (!skip_rx_queue(bp, index)) {
4569                /* fastpath rx rings: rx_buf rx_desc rx_comp */
4570                bnx2x_fp(bp, index, rx_buf_ring) =
4571                        kcalloc(NUM_RX_BD, sizeof(struct sw_rx_bd), GFP_KERNEL);
4572                if (!bnx2x_fp(bp, index, rx_buf_ring))
4573                        goto alloc_mem_err;
4574                bnx2x_fp(bp, index, rx_desc_ring) =
4575                        BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_desc_mapping),
4576                                        sizeof(struct eth_rx_bd) * NUM_RX_BD);
4577                if (!bnx2x_fp(bp, index, rx_desc_ring))
4578                        goto alloc_mem_err;
4579
4580                /* Seed all CQEs by 1s */
4581                bnx2x_fp(bp, index, rx_comp_ring) =
4582                        BNX2X_PCI_FALLOC(&bnx2x_fp(bp, index, rx_comp_mapping),
4583                                         sizeof(struct eth_fast_path_rx_cqe) * NUM_RCQ_BD);
4584                if (!bnx2x_fp(bp, index, rx_comp_ring))
4585                        goto alloc_mem_err;
4586
4587                /* SGE ring */
4588                bnx2x_fp(bp, index, rx_page_ring) =
4589                        kcalloc(NUM_RX_SGE, sizeof(struct sw_rx_page),
4590                                GFP_KERNEL);
4591                if (!bnx2x_fp(bp, index, rx_page_ring))
4592                        goto alloc_mem_err;
4593                bnx2x_fp(bp, index, rx_sge_ring) =
4594                        BNX2X_PCI_ALLOC(&bnx2x_fp(bp, index, rx_sge_mapping),
4595                                        BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4596                if (!bnx2x_fp(bp, index, rx_sge_ring))
4597                        goto alloc_mem_err;
4598                /* RX BD ring */
4599                bnx2x_set_next_page_rx_bd(fp);
4600
4601                /* CQ ring */
4602                bnx2x_set_next_page_rx_cq(fp);
4603
4604                /* BDs */
4605                ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4606                if (ring_size < rx_ring_size)
4607                        goto alloc_mem_err;
4608        }
4609
4610        return 0;
4611
4612/* handles low memory cases */
4613alloc_mem_err:
4614        BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4615                                                index, ring_size);
4616        /* FW will drop all packets if queue is not big enough,
4617         * In these cases we disable the queue
4618         * Min size is different for OOO, TPA and non-TPA queues
4619         */
4620        if (ring_size < (fp->mode == TPA_MODE_DISABLED ?
4621                                MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4622                        /* release memory allocated for this queue */
4623                        bnx2x_free_fp_mem_at(bp, index);
4624                        return -ENOMEM;
4625        }
4626        return 0;
4627}
4628
4629static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4630{
4631        if (!NO_FCOE(bp))
4632                /* FCoE */
4633                if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4634                        /* we will fail load process instead of mark
4635                         * NO_FCOE_FLAG
4636                         */
4637                        return -ENOMEM;
4638
4639        return 0;
4640}
4641
4642static int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4643{
4644        int i;
4645
4646        /* 1. Allocate FP for leading - fatal if error
4647         * 2. Allocate RSS - fix number of queues if error
4648         */
4649
4650        /* leading */
4651        if (bnx2x_alloc_fp_mem_at(bp, 0))
4652                return -ENOMEM;
4653
4654        /* RSS */
4655        for_each_nondefault_eth_queue(bp, i)
4656                if (bnx2x_alloc_fp_mem_at(bp, i))
4657                        break;
4658
4659        /* handle memory failures */
4660        if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4661                int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4662
4663                WARN_ON(delta < 0);
4664                bnx2x_shrink_eth_fp(bp, delta);
4665                if (CNIC_SUPPORT(bp))
4666                        /* move non eth FPs next to last eth FP
4667                         * must be done in that order
4668                         * FCOE_IDX < FWD_IDX < OOO_IDX
4669                         */
4670
4671                        /* move FCoE fp even NO_FCOE_FLAG is on */
4672                        bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4673                bp->num_ethernet_queues -= delta;
4674                bp->num_queues = bp->num_ethernet_queues +
4675                                 bp->num_cnic_queues;
4676                BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4677                          bp->num_queues + delta, bp->num_queues);
4678        }
4679
4680        return 0;
4681}
4682
4683void bnx2x_free_mem_bp(struct bnx2x *bp)
4684{
4685        int i;
4686
4687        for (i = 0; i < bp->fp_array_size; i++)
4688                kfree(bp->fp[i].tpa_info);
4689        kfree(bp->fp);
4690        kfree(bp->sp_objs);
4691        kfree(bp->fp_stats);
4692        kfree(bp->bnx2x_txq);
4693        kfree(bp->msix_table);
4694        kfree(bp->ilt);
4695}
4696
4697int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4698{
4699        struct bnx2x_fastpath *fp;
4700        struct msix_entry *tbl;
4701        struct bnx2x_ilt *ilt;
4702        int msix_table_size = 0;
4703        int fp_array_size, txq_array_size;
4704        int i;
4705
4706        /*
4707         * The biggest MSI-X table we might need is as a maximum number of fast
4708         * path IGU SBs plus default SB (for PF only).
4709         */
4710        msix_table_size = bp->igu_sb_cnt;
4711        if (IS_PF(bp))
4712                msix_table_size++;
4713        BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4714
4715        /* fp array: RSS plus CNIC related L2 queues */
4716        fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4717        bp->fp_array_size = fp_array_size;
4718        BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4719
4720        fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4721        if (!fp)
4722                goto alloc_err;
4723        for (i = 0; i < bp->fp_array_size; i++) {
4724                fp[i].tpa_info =
4725                        kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4726                                sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4727                if (!(fp[i].tpa_info))
4728                        goto alloc_err;
4729        }
4730
4731        bp->fp = fp;
4732
4733        /* allocate sp objs */
4734        bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4735                              GFP_KERNEL);
4736        if (!bp->sp_objs)
4737                goto alloc_err;
4738
4739        /* allocate fp_stats */
4740        bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4741                               GFP_KERNEL);
4742        if (!bp->fp_stats)
4743                goto alloc_err;
4744
4745        /* Allocate memory for the transmission queues array */
4746        txq_array_size =
4747                BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4748        BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4749
4750        bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4751                                GFP_KERNEL);
4752        if (!bp->bnx2x_txq)
4753                goto alloc_err;
4754
4755        /* msix table */
4756        tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4757        if (!tbl)
4758                goto alloc_err;
4759        bp->msix_table = tbl;
4760
4761        /* ilt */
4762        ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4763        if (!ilt)
4764                goto alloc_err;
4765        bp->ilt = ilt;
4766
4767        return 0;
4768alloc_err:
4769        bnx2x_free_mem_bp(bp);
4770        return -ENOMEM;
4771}
4772
4773int bnx2x_reload_if_running(struct net_device *dev)
4774{
4775        struct bnx2x *bp = netdev_priv(dev);
4776
4777        if (unlikely(!netif_running(dev)))
4778                return 0;
4779
4780        bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4781        return bnx2x_nic_load(bp, LOAD_NORMAL);
4782}
4783
4784int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4785{
4786        u32 sel_phy_idx = 0;
4787        if (bp->link_params.num_phys <= 1)
4788                return INT_PHY;
4789
4790        if (bp->link_vars.link_up) {
4791                sel_phy_idx = EXT_PHY1;
4792                /* In case link is SERDES, check if the EXT_PHY2 is the one */
4793                if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4794                    (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4795                        sel_phy_idx = EXT_PHY2;
4796        } else {
4797
4798                switch (bnx2x_phy_selection(&bp->link_params)) {
4799                case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4800                case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4801                case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4802                       sel_phy_idx = EXT_PHY1;
4803                       break;
4804                case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4805                case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4806                       sel_phy_idx = EXT_PHY2;
4807                       break;
4808                }
4809        }
4810
4811        return sel_phy_idx;
4812}
4813int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4814{
4815        u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4816        /*
4817         * The selected activated PHY is always after swapping (in case PHY
4818         * swapping is enabled). So when swapping is enabled, we need to reverse
4819         * the configuration
4820         */
4821
4822        if (bp->link_params.multi_phy_config &
4823            PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4824                if (sel_phy_idx == EXT_PHY1)
4825                        sel_phy_idx = EXT_PHY2;
4826                else if (sel_phy_idx == EXT_PHY2)
4827                        sel_phy_idx = EXT_PHY1;
4828        }
4829        return LINK_CONFIG_IDX(sel_phy_idx);
4830}
4831
4832#ifdef NETDEV_FCOE_WWNN
4833int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4834{
4835        struct bnx2x *bp = netdev_priv(dev);
4836        struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4837
4838        switch (type) {
4839        case NETDEV_FCOE_WWNN:
4840                *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4841                                cp->fcoe_wwn_node_name_lo);
4842                break;
4843        case NETDEV_FCOE_WWPN:
4844                *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4845                                cp->fcoe_wwn_port_name_lo);
4846                break;
4847        default:
4848                BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4849                return -EINVAL;
4850        }
4851
4852        return 0;
4853}
4854#endif
4855
4856/* called with rtnl_lock */
4857int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4858{
4859        struct bnx2x *bp = netdev_priv(dev);
4860
4861        if (pci_num_vf(bp->pdev)) {
4862                DP(BNX2X_MSG_IOV, "VFs are enabled, can not change MTU\n");
4863                return -EPERM;
4864        }
4865
4866        if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4867                BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4868                return -EAGAIN;
4869        }
4870
4871        /* This does not race with packet allocation
4872         * because the actual alloc size is
4873         * only updated as part of load
4874         */
4875        dev->mtu = new_mtu;
4876
4877        if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
4878                SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
4879
4880        return bnx2x_reload_if_running(dev);
4881}
4882
4883netdev_features_t bnx2x_fix_features(struct net_device *dev,
4884                                     netdev_features_t features)
4885{
4886        struct bnx2x *bp = netdev_priv(dev);
4887
4888        if (pci_num_vf(bp->pdev)) {
4889                netdev_features_t changed = dev->features ^ features;
4890
4891                /* Revert the requested changes in features if they
4892                 * would require internal reload of PF in bnx2x_set_features().
4893                 */
4894                if (!(features & NETIF_F_RXCSUM) && !bp->disable_tpa) {
4895                        features &= ~NETIF_F_RXCSUM;
4896                        features |= dev->features & NETIF_F_RXCSUM;
4897                }
4898
4899                if (changed & NETIF_F_LOOPBACK) {
4900                        features &= ~NETIF_F_LOOPBACK;
4901                        features |= dev->features & NETIF_F_LOOPBACK;
4902                }
4903        }
4904
4905        /* TPA requires Rx CSUM offloading */
4906        if (!(features & NETIF_F_RXCSUM)) {
4907                features &= ~NETIF_F_LRO;
4908                features &= ~NETIF_F_GRO;
4909        }
4910
4911        return features;
4912}
4913
4914int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4915{
4916        struct bnx2x *bp = netdev_priv(dev);
4917        netdev_features_t changes = features ^ dev->features;
4918        bool bnx2x_reload = false;
4919        int rc;
4920
4921        /* VFs or non SRIOV PFs should be able to change loopback feature */
4922        if (!pci_num_vf(bp->pdev)) {
4923                if (features & NETIF_F_LOOPBACK) {
4924                        if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4925                                bp->link_params.loopback_mode = LOOPBACK_BMAC;
4926                                bnx2x_reload = true;
4927                        }
4928                } else {
4929                        if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4930                                bp->link_params.loopback_mode = LOOPBACK_NONE;
4931                                bnx2x_reload = true;
4932                        }
4933                }
4934        }
4935
4936        /* if GRO is changed while LRO is enabled, don't force a reload */
4937        if ((changes & NETIF_F_GRO) && (features & NETIF_F_LRO))
4938                changes &= ~NETIF_F_GRO;
4939
4940        /* if GRO is changed while HW TPA is off, don't force a reload */
4941        if ((changes & NETIF_F_GRO) && bp->disable_tpa)
4942                changes &= ~NETIF_F_GRO;
4943
4944        if (changes)
4945                bnx2x_reload = true;
4946
4947        if (bnx2x_reload) {
4948                if (bp->recovery_state == BNX2X_RECOVERY_DONE) {
4949                        dev->features = features;
4950                        rc = bnx2x_reload_if_running(dev);
4951                        return rc ? rc : 1;
4952                }
4953                /* else: bnx2x_nic_load() will be called at end of recovery */
4954        }
4955
4956        return 0;
4957}
4958
4959void bnx2x_tx_timeout(struct net_device *dev)
4960{
4961        struct bnx2x *bp = netdev_priv(dev);
4962
4963#ifdef BNX2X_STOP_ON_ERROR
4964        if (!bp->panic)
4965                bnx2x_panic();
4966#endif
4967
4968        /* This allows the netif to be shutdown gracefully before resetting */
4969        bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_TX_TIMEOUT, 0);
4970}
4971
4972int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4973{
4974        struct net_device *dev = pci_get_drvdata(pdev);
4975        struct bnx2x *bp;
4976
4977        if (!dev) {
4978                dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4979                return -ENODEV;
4980        }
4981        bp = netdev_priv(dev);
4982
4983        rtnl_lock();
4984
4985        pci_save_state(pdev);
4986
4987        if (!netif_running(dev)) {
4988                rtnl_unlock();
4989                return 0;
4990        }
4991
4992        netif_device_detach(dev);
4993
4994        bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4995
4996        bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4997
4998        rtnl_unlock();
4999
5000        return 0;
5001}
5002
5003int bnx2x_resume(struct pci_dev *pdev)
5004{
5005        struct net_device *dev = pci_get_drvdata(pdev);
5006        struct bnx2x *bp;
5007        int rc;
5008
5009        if (!dev) {
5010                dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
5011                return -ENODEV;
5012        }
5013        bp = netdev_priv(dev);
5014
5015        if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
5016                BNX2X_ERR("Handling parity error recovery. Try again later\n");
5017                return -EAGAIN;
5018        }
5019
5020        rtnl_lock();
5021
5022        pci_restore_state(pdev);
5023
5024        if (!netif_running(dev)) {
5025                rtnl_unlock();
5026                return 0;
5027        }
5028
5029        bnx2x_set_power_state(bp, PCI_D0);
5030        netif_device_attach(dev);
5031
5032        rc = bnx2x_nic_load(bp, LOAD_OPEN);
5033
5034        rtnl_unlock();
5035
5036        return rc;
5037}
5038
5039void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
5040                              u32 cid)
5041{
5042        if (!cxt) {
5043                BNX2X_ERR("bad context pointer %p\n", cxt);
5044                return;
5045        }
5046
5047        /* ustorm cxt validation */
5048        cxt->ustorm_ag_context.cdu_usage =
5049                CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5050                        CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
5051        /* xcontext validation */
5052        cxt->xstorm_ag_context.cdu_reserved =
5053                CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
5054                        CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
5055}
5056
5057static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
5058                                    u8 fw_sb_id, u8 sb_index,
5059                                    u8 ticks)
5060{
5061        u32 addr = BAR_CSTRORM_INTMEM +
5062                   CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
5063        REG_WR8(bp, addr, ticks);
5064        DP(NETIF_MSG_IFUP,
5065           "port %x fw_sb_id %d sb_index %d ticks %d\n",
5066           port, fw_sb_id, sb_index, ticks);
5067}
5068
5069static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
5070                                    u16 fw_sb_id, u8 sb_index,
5071                                    u8 disable)
5072{
5073        u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
5074        u32 addr = BAR_CSTRORM_INTMEM +
5075                   CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
5076        u8 flags = REG_RD8(bp, addr);
5077        /* clear and set */
5078        flags &= ~HC_INDEX_DATA_HC_ENABLED;
5079        flags |= enable_flag;
5080        REG_WR8(bp, addr, flags);
5081        DP(NETIF_MSG_IFUP,
5082           "port %x fw_sb_id %d sb_index %d disable %d\n",
5083           port, fw_sb_id, sb_index, disable);
5084}
5085
5086void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
5087                                    u8 sb_index, u8 disable, u16 usec)
5088{
5089        int port = BP_PORT(bp);
5090        u8 ticks = usec / BNX2X_BTR;
5091
5092        storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
5093
5094        disable = disable ? 1 : (usec ? 0 : 1);
5095        storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
5096}
5097
5098void bnx2x_schedule_sp_rtnl(struct bnx2x *bp, enum sp_rtnl_flag flag,
5099                            u32 verbose)
5100{
5101        smp_mb__before_atomic();
5102        set_bit(flag, &bp->sp_rtnl_state);
5103        smp_mb__after_atomic();
5104        DP((BNX2X_MSG_SP | verbose), "Scheduling sp_rtnl task [Flag: %d]\n",
5105           flag);
5106        schedule_delayed_work(&bp->sp_rtnl_task, 0);
5107}
5108