linux/drivers/net/ethernet/broadcom/cnic.c
<<
>>
Prefs
   1/* cnic.c: QLogic CNIC core network driver.
   2 *
   3 * Copyright (c) 2006-2014 Broadcom Corporation
   4 * Copyright (c) 2014-2015 QLogic Corporation
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation.
   9 *
  10 * Original skeleton written by: John(Zongxi) Chen (zongxi@broadcom.com)
  11 * Previously modified and maintained by: Michael Chan <mchan@broadcom.com>
  12 * Maintained By: Dept-HSGLinuxNICDev@qlogic.com
  13 */
  14
  15#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  16
  17#include <linux/module.h>
  18
  19#include <linux/kernel.h>
  20#include <linux/errno.h>
  21#include <linux/list.h>
  22#include <linux/slab.h>
  23#include <linux/pci.h>
  24#include <linux/init.h>
  25#include <linux/netdevice.h>
  26#include <linux/uio_driver.h>
  27#include <linux/in.h>
  28#include <linux/dma-mapping.h>
  29#include <linux/delay.h>
  30#include <linux/ethtool.h>
  31#include <linux/if_vlan.h>
  32#include <linux/prefetch.h>
  33#include <linux/random.h>
  34#if IS_ENABLED(CONFIG_VLAN_8021Q)
  35#define BCM_VLAN 1
  36#endif
  37#include <net/ip.h>
  38#include <net/tcp.h>
  39#include <net/route.h>
  40#include <net/ipv6.h>
  41#include <net/ip6_route.h>
  42#include <net/ip6_checksum.h>
  43#include <scsi/iscsi_if.h>
  44
  45#define BCM_CNIC        1
  46#include "cnic_if.h"
  47#include "bnx2.h"
  48#include "bnx2x/bnx2x.h"
  49#include "bnx2x/bnx2x_reg.h"
  50#include "bnx2x/bnx2x_fw_defs.h"
  51#include "bnx2x/bnx2x_hsi.h"
  52#include "../../../scsi/bnx2i/57xx_iscsi_constants.h"
  53#include "../../../scsi/bnx2i/57xx_iscsi_hsi.h"
  54#include "../../../scsi/bnx2fc/bnx2fc_constants.h"
  55#include "cnic.h"
  56#include "cnic_defs.h"
  57
  58#define CNIC_MODULE_NAME        "cnic"
  59
  60static char version[] =
  61        "QLogic " CNIC_MODULE_NAME "Driver v" CNIC_MODULE_VERSION " (" CNIC_MODULE_RELDATE ")\n";
  62
  63MODULE_AUTHOR("Michael Chan <mchan@broadcom.com> and John(Zongxi) "
  64              "Chen (zongxi@broadcom.com");
  65MODULE_DESCRIPTION("QLogic cnic Driver");
  66MODULE_LICENSE("GPL");
  67MODULE_VERSION(CNIC_MODULE_VERSION);
  68
  69/* cnic_dev_list modifications are protected by both rtnl and cnic_dev_lock */
  70static LIST_HEAD(cnic_dev_list);
  71static LIST_HEAD(cnic_udev_list);
  72static DEFINE_RWLOCK(cnic_dev_lock);
  73static DEFINE_MUTEX(cnic_lock);
  74
  75static struct cnic_ulp_ops __rcu *cnic_ulp_tbl[MAX_CNIC_ULP_TYPE];
  76
  77/* helper function, assuming cnic_lock is held */
  78static inline struct cnic_ulp_ops *cnic_ulp_tbl_prot(int type)
  79{
  80        return rcu_dereference_protected(cnic_ulp_tbl[type],
  81                                         lockdep_is_held(&cnic_lock));
  82}
  83
  84static int cnic_service_bnx2(void *, void *);
  85static int cnic_service_bnx2x(void *, void *);
  86static int cnic_ctl(void *, struct cnic_ctl_info *);
  87
  88static struct cnic_ops cnic_bnx2_ops = {
  89        .cnic_owner     = THIS_MODULE,
  90        .cnic_handler   = cnic_service_bnx2,
  91        .cnic_ctl       = cnic_ctl,
  92};
  93
  94static struct cnic_ops cnic_bnx2x_ops = {
  95        .cnic_owner     = THIS_MODULE,
  96        .cnic_handler   = cnic_service_bnx2x,
  97        .cnic_ctl       = cnic_ctl,
  98};
  99
 100static struct workqueue_struct *cnic_wq;
 101
 102static void cnic_shutdown_rings(struct cnic_dev *);
 103static void cnic_init_rings(struct cnic_dev *);
 104static int cnic_cm_set_pg(struct cnic_sock *);
 105
 106static int cnic_uio_open(struct uio_info *uinfo, struct inode *inode)
 107{
 108        struct cnic_uio_dev *udev = uinfo->priv;
 109        struct cnic_dev *dev;
 110
 111        if (!capable(CAP_NET_ADMIN))
 112                return -EPERM;
 113
 114        if (udev->uio_dev != -1)
 115                return -EBUSY;
 116
 117        rtnl_lock();
 118        dev = udev->dev;
 119
 120        if (!dev || !test_bit(CNIC_F_CNIC_UP, &dev->flags)) {
 121                rtnl_unlock();
 122                return -ENODEV;
 123        }
 124
 125        udev->uio_dev = iminor(inode);
 126
 127        cnic_shutdown_rings(dev);
 128        cnic_init_rings(dev);
 129        rtnl_unlock();
 130
 131        return 0;
 132}
 133
 134static int cnic_uio_close(struct uio_info *uinfo, struct inode *inode)
 135{
 136        struct cnic_uio_dev *udev = uinfo->priv;
 137
 138        udev->uio_dev = -1;
 139        return 0;
 140}
 141
 142static inline void cnic_hold(struct cnic_dev *dev)
 143{
 144        atomic_inc(&dev->ref_count);
 145}
 146
 147static inline void cnic_put(struct cnic_dev *dev)
 148{
 149        atomic_dec(&dev->ref_count);
 150}
 151
 152static inline void csk_hold(struct cnic_sock *csk)
 153{
 154        atomic_inc(&csk->ref_count);
 155}
 156
 157static inline void csk_put(struct cnic_sock *csk)
 158{
 159        atomic_dec(&csk->ref_count);
 160}
 161
 162static struct cnic_dev *cnic_from_netdev(struct net_device *netdev)
 163{
 164        struct cnic_dev *cdev;
 165
 166        read_lock(&cnic_dev_lock);
 167        list_for_each_entry(cdev, &cnic_dev_list, list) {
 168                if (netdev == cdev->netdev) {
 169                        cnic_hold(cdev);
 170                        read_unlock(&cnic_dev_lock);
 171                        return cdev;
 172                }
 173        }
 174        read_unlock(&cnic_dev_lock);
 175        return NULL;
 176}
 177
 178static inline void ulp_get(struct cnic_ulp_ops *ulp_ops)
 179{
 180        atomic_inc(&ulp_ops->ref_count);
 181}
 182
 183static inline void ulp_put(struct cnic_ulp_ops *ulp_ops)
 184{
 185        atomic_dec(&ulp_ops->ref_count);
 186}
 187
 188static void cnic_ctx_wr(struct cnic_dev *dev, u32 cid_addr, u32 off, u32 val)
 189{
 190        struct cnic_local *cp = dev->cnic_priv;
 191        struct cnic_eth_dev *ethdev = cp->ethdev;
 192        struct drv_ctl_info info;
 193        struct drv_ctl_io *io = &info.data.io;
 194
 195        memset(&info, 0, sizeof(struct drv_ctl_info));
 196        info.cmd = DRV_CTL_CTX_WR_CMD;
 197        io->cid_addr = cid_addr;
 198        io->offset = off;
 199        io->data = val;
 200        ethdev->drv_ctl(dev->netdev, &info);
 201}
 202
 203static void cnic_ctx_tbl_wr(struct cnic_dev *dev, u32 off, dma_addr_t addr)
 204{
 205        struct cnic_local *cp = dev->cnic_priv;
 206        struct cnic_eth_dev *ethdev = cp->ethdev;
 207        struct drv_ctl_info info;
 208        struct drv_ctl_io *io = &info.data.io;
 209
 210        memset(&info, 0, sizeof(struct drv_ctl_info));
 211        info.cmd = DRV_CTL_CTXTBL_WR_CMD;
 212        io->offset = off;
 213        io->dma_addr = addr;
 214        ethdev->drv_ctl(dev->netdev, &info);
 215}
 216
 217static void cnic_ring_ctl(struct cnic_dev *dev, u32 cid, u32 cl_id, int start)
 218{
 219        struct cnic_local *cp = dev->cnic_priv;
 220        struct cnic_eth_dev *ethdev = cp->ethdev;
 221        struct drv_ctl_info info;
 222        struct drv_ctl_l2_ring *ring = &info.data.ring;
 223
 224        memset(&info, 0, sizeof(struct drv_ctl_info));
 225        if (start)
 226                info.cmd = DRV_CTL_START_L2_CMD;
 227        else
 228                info.cmd = DRV_CTL_STOP_L2_CMD;
 229
 230        ring->cid = cid;
 231        ring->client_id = cl_id;
 232        ethdev->drv_ctl(dev->netdev, &info);
 233}
 234
 235static void cnic_reg_wr_ind(struct cnic_dev *dev, u32 off, u32 val)
 236{
 237        struct cnic_local *cp = dev->cnic_priv;
 238        struct cnic_eth_dev *ethdev = cp->ethdev;
 239        struct drv_ctl_info info;
 240        struct drv_ctl_io *io = &info.data.io;
 241
 242        memset(&info, 0, sizeof(struct drv_ctl_info));
 243        info.cmd = DRV_CTL_IO_WR_CMD;
 244        io->offset = off;
 245        io->data = val;
 246        ethdev->drv_ctl(dev->netdev, &info);
 247}
 248
 249static u32 cnic_reg_rd_ind(struct cnic_dev *dev, u32 off)
 250{
 251        struct cnic_local *cp = dev->cnic_priv;
 252        struct cnic_eth_dev *ethdev = cp->ethdev;
 253        struct drv_ctl_info info;
 254        struct drv_ctl_io *io = &info.data.io;
 255
 256        memset(&info, 0, sizeof(struct drv_ctl_info));
 257        info.cmd = DRV_CTL_IO_RD_CMD;
 258        io->offset = off;
 259        ethdev->drv_ctl(dev->netdev, &info);
 260        return io->data;
 261}
 262
 263static void cnic_ulp_ctl(struct cnic_dev *dev, int ulp_type, bool reg, int state)
 264{
 265        struct cnic_local *cp = dev->cnic_priv;
 266        struct cnic_eth_dev *ethdev = cp->ethdev;
 267        struct drv_ctl_info info;
 268        struct fcoe_capabilities *fcoe_cap =
 269                &info.data.register_data.fcoe_features;
 270
 271        memset(&info, 0, sizeof(struct drv_ctl_info));
 272        if (reg) {
 273                info.cmd = DRV_CTL_ULP_REGISTER_CMD;
 274                if (ulp_type == CNIC_ULP_FCOE && dev->fcoe_cap)
 275                        memcpy(fcoe_cap, dev->fcoe_cap, sizeof(*fcoe_cap));
 276        } else {
 277                info.cmd = DRV_CTL_ULP_UNREGISTER_CMD;
 278        }
 279
 280        info.data.ulp_type = ulp_type;
 281        info.drv_state = state;
 282        ethdev->drv_ctl(dev->netdev, &info);
 283}
 284
 285static int cnic_in_use(struct cnic_sock *csk)
 286{
 287        return test_bit(SK_F_INUSE, &csk->flags);
 288}
 289
 290static void cnic_spq_completion(struct cnic_dev *dev, int cmd, u32 count)
 291{
 292        struct cnic_local *cp = dev->cnic_priv;
 293        struct cnic_eth_dev *ethdev = cp->ethdev;
 294        struct drv_ctl_info info;
 295
 296        memset(&info, 0, sizeof(struct drv_ctl_info));
 297        info.cmd = cmd;
 298        info.data.credit.credit_count = count;
 299        ethdev->drv_ctl(dev->netdev, &info);
 300}
 301
 302static int cnic_get_l5_cid(struct cnic_local *cp, u32 cid, u32 *l5_cid)
 303{
 304        u32 i;
 305
 306        if (!cp->ctx_tbl)
 307                return -EINVAL;
 308
 309        for (i = 0; i < cp->max_cid_space; i++) {
 310                if (cp->ctx_tbl[i].cid == cid) {
 311                        *l5_cid = i;
 312                        return 0;
 313                }
 314        }
 315        return -EINVAL;
 316}
 317
 318static int cnic_send_nlmsg(struct cnic_local *cp, u32 type,
 319                           struct cnic_sock *csk)
 320{
 321        struct iscsi_path path_req;
 322        char *buf = NULL;
 323        u16 len = 0;
 324        u32 msg_type = ISCSI_KEVENT_IF_DOWN;
 325        struct cnic_ulp_ops *ulp_ops;
 326        struct cnic_uio_dev *udev = cp->udev;
 327        int rc = 0, retry = 0;
 328
 329        if (!udev || udev->uio_dev == -1)
 330                return -ENODEV;
 331
 332        if (csk) {
 333                len = sizeof(path_req);
 334                buf = (char *) &path_req;
 335                memset(&path_req, 0, len);
 336
 337                msg_type = ISCSI_KEVENT_PATH_REQ;
 338                path_req.handle = (u64) csk->l5_cid;
 339                if (test_bit(SK_F_IPV6, &csk->flags)) {
 340                        memcpy(&path_req.dst.v6_addr, &csk->dst_ip[0],
 341                               sizeof(struct in6_addr));
 342                        path_req.ip_addr_len = 16;
 343                } else {
 344                        memcpy(&path_req.dst.v4_addr, &csk->dst_ip[0],
 345                               sizeof(struct in_addr));
 346                        path_req.ip_addr_len = 4;
 347                }
 348                path_req.vlan_id = csk->vlan_id;
 349                path_req.pmtu = csk->mtu;
 350        }
 351
 352        while (retry < 3) {
 353                rc = 0;
 354                rcu_read_lock();
 355                ulp_ops = rcu_dereference(cp->ulp_ops[CNIC_ULP_ISCSI]);
 356                if (ulp_ops)
 357                        rc = ulp_ops->iscsi_nl_send_msg(
 358                                cp->ulp_handle[CNIC_ULP_ISCSI],
 359                                msg_type, buf, len);
 360                rcu_read_unlock();
 361                if (rc == 0 || msg_type != ISCSI_KEVENT_PATH_REQ)
 362                        break;
 363
 364                msleep(100);
 365                retry++;
 366        }
 367        return rc;
 368}
 369
 370static void cnic_cm_upcall(struct cnic_local *, struct cnic_sock *, u8);
 371
 372static int cnic_iscsi_nl_msg_recv(struct cnic_dev *dev, u32 msg_type,
 373                                  char *buf, u16 len)
 374{
 375        int rc = -EINVAL;
 376
 377        switch (msg_type) {
 378        case ISCSI_UEVENT_PATH_UPDATE: {
 379                struct cnic_local *cp;
 380                u32 l5_cid;
 381                struct cnic_sock *csk;
 382                struct iscsi_path *path_resp;
 383
 384                if (len < sizeof(*path_resp))
 385                        break;
 386
 387                path_resp = (struct iscsi_path *) buf;
 388                cp = dev->cnic_priv;
 389                l5_cid = (u32) path_resp->handle;
 390                if (l5_cid >= MAX_CM_SK_TBL_SZ)
 391                        break;
 392
 393                if (!rcu_access_pointer(cp->ulp_ops[CNIC_ULP_L4])) {
 394                        rc = -ENODEV;
 395                        break;
 396                }
 397                csk = &cp->csk_tbl[l5_cid];
 398                csk_hold(csk);
 399                if (cnic_in_use(csk) &&
 400                    test_bit(SK_F_CONNECT_START, &csk->flags)) {
 401
 402                        csk->vlan_id = path_resp->vlan_id;
 403
 404                        memcpy(csk->ha, path_resp->mac_addr, ETH_ALEN);
 405                        if (test_bit(SK_F_IPV6, &csk->flags))
 406                                memcpy(&csk->src_ip[0], &path_resp->src.v6_addr,
 407                                       sizeof(struct in6_addr));
 408                        else
 409                                memcpy(&csk->src_ip[0], &path_resp->src.v4_addr,
 410                                       sizeof(struct in_addr));
 411
 412                        if (is_valid_ether_addr(csk->ha)) {
 413                                cnic_cm_set_pg(csk);
 414                        } else if (!test_bit(SK_F_OFFLD_SCHED, &csk->flags) &&
 415                                !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) {
 416
 417                                cnic_cm_upcall(cp, csk,
 418                                        L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE);
 419                                clear_bit(SK_F_CONNECT_START, &csk->flags);
 420                        }
 421                }
 422                csk_put(csk);
 423                rc = 0;
 424        }
 425        }
 426
 427        return rc;
 428}
 429
 430static int cnic_offld_prep(struct cnic_sock *csk)
 431{
 432        if (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags))
 433                return 0;
 434
 435        if (!test_bit(SK_F_CONNECT_START, &csk->flags)) {
 436                clear_bit(SK_F_OFFLD_SCHED, &csk->flags);
 437                return 0;
 438        }
 439
 440        return 1;
 441}
 442
 443static int cnic_close_prep(struct cnic_sock *csk)
 444{
 445        clear_bit(SK_F_CONNECT_START, &csk->flags);
 446        smp_mb__after_atomic();
 447
 448        if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) {
 449                while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags))
 450                        msleep(1);
 451
 452                return 1;
 453        }
 454        return 0;
 455}
 456
 457static int cnic_abort_prep(struct cnic_sock *csk)
 458{
 459        clear_bit(SK_F_CONNECT_START, &csk->flags);
 460        smp_mb__after_atomic();
 461
 462        while (test_and_set_bit(SK_F_OFFLD_SCHED, &csk->flags))
 463                msleep(1);
 464
 465        if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) {
 466                csk->state = L4_KCQE_OPCODE_VALUE_RESET_COMP;
 467                return 1;
 468        }
 469
 470        return 0;
 471}
 472
 473int cnic_register_driver(int ulp_type, struct cnic_ulp_ops *ulp_ops)
 474{
 475        struct cnic_dev *dev;
 476
 477        if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) {
 478                pr_err("%s: Bad type %d\n", __func__, ulp_type);
 479                return -EINVAL;
 480        }
 481        mutex_lock(&cnic_lock);
 482        if (cnic_ulp_tbl_prot(ulp_type)) {
 483                pr_err("%s: Type %d has already been registered\n",
 484                       __func__, ulp_type);
 485                mutex_unlock(&cnic_lock);
 486                return -EBUSY;
 487        }
 488
 489        read_lock(&cnic_dev_lock);
 490        list_for_each_entry(dev, &cnic_dev_list, list) {
 491                struct cnic_local *cp = dev->cnic_priv;
 492
 493                clear_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]);
 494        }
 495        read_unlock(&cnic_dev_lock);
 496
 497        atomic_set(&ulp_ops->ref_count, 0);
 498        rcu_assign_pointer(cnic_ulp_tbl[ulp_type], ulp_ops);
 499        mutex_unlock(&cnic_lock);
 500
 501        /* Prevent race conditions with netdev_event */
 502        rtnl_lock();
 503        list_for_each_entry(dev, &cnic_dev_list, list) {
 504                struct cnic_local *cp = dev->cnic_priv;
 505
 506                if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]))
 507                        ulp_ops->cnic_init(dev);
 508        }
 509        rtnl_unlock();
 510
 511        return 0;
 512}
 513
 514int cnic_unregister_driver(int ulp_type)
 515{
 516        struct cnic_dev *dev;
 517        struct cnic_ulp_ops *ulp_ops;
 518        int i = 0;
 519
 520        if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) {
 521                pr_err("%s: Bad type %d\n", __func__, ulp_type);
 522                return -EINVAL;
 523        }
 524        mutex_lock(&cnic_lock);
 525        ulp_ops = cnic_ulp_tbl_prot(ulp_type);
 526        if (!ulp_ops) {
 527                pr_err("%s: Type %d has not been registered\n",
 528                       __func__, ulp_type);
 529                goto out_unlock;
 530        }
 531        read_lock(&cnic_dev_lock);
 532        list_for_each_entry(dev, &cnic_dev_list, list) {
 533                struct cnic_local *cp = dev->cnic_priv;
 534
 535                if (rcu_access_pointer(cp->ulp_ops[ulp_type])) {
 536                        pr_err("%s: Type %d still has devices registered\n",
 537                               __func__, ulp_type);
 538                        read_unlock(&cnic_dev_lock);
 539                        goto out_unlock;
 540                }
 541        }
 542        read_unlock(&cnic_dev_lock);
 543
 544        RCU_INIT_POINTER(cnic_ulp_tbl[ulp_type], NULL);
 545
 546        mutex_unlock(&cnic_lock);
 547        synchronize_rcu();
 548        while ((atomic_read(&ulp_ops->ref_count) != 0) && (i < 20)) {
 549                msleep(100);
 550                i++;
 551        }
 552
 553        if (atomic_read(&ulp_ops->ref_count) != 0)
 554                pr_warn("%s: Failed waiting for ref count to go to zero\n",
 555                        __func__);
 556        return 0;
 557
 558out_unlock:
 559        mutex_unlock(&cnic_lock);
 560        return -EINVAL;
 561}
 562
 563static int cnic_start_hw(struct cnic_dev *);
 564static void cnic_stop_hw(struct cnic_dev *);
 565
 566static int cnic_register_device(struct cnic_dev *dev, int ulp_type,
 567                                void *ulp_ctx)
 568{
 569        struct cnic_local *cp = dev->cnic_priv;
 570        struct cnic_ulp_ops *ulp_ops;
 571
 572        if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) {
 573                pr_err("%s: Bad type %d\n", __func__, ulp_type);
 574                return -EINVAL;
 575        }
 576        mutex_lock(&cnic_lock);
 577        if (cnic_ulp_tbl_prot(ulp_type) == NULL) {
 578                pr_err("%s: Driver with type %d has not been registered\n",
 579                       __func__, ulp_type);
 580                mutex_unlock(&cnic_lock);
 581                return -EAGAIN;
 582        }
 583        if (rcu_access_pointer(cp->ulp_ops[ulp_type])) {
 584                pr_err("%s: Type %d has already been registered to this device\n",
 585                       __func__, ulp_type);
 586                mutex_unlock(&cnic_lock);
 587                return -EBUSY;
 588        }
 589
 590        clear_bit(ULP_F_START, &cp->ulp_flags[ulp_type]);
 591        cp->ulp_handle[ulp_type] = ulp_ctx;
 592        ulp_ops = cnic_ulp_tbl_prot(ulp_type);
 593        rcu_assign_pointer(cp->ulp_ops[ulp_type], ulp_ops);
 594        cnic_hold(dev);
 595
 596        if (test_bit(CNIC_F_CNIC_UP, &dev->flags))
 597                if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[ulp_type]))
 598                        ulp_ops->cnic_start(cp->ulp_handle[ulp_type]);
 599
 600        mutex_unlock(&cnic_lock);
 601
 602        cnic_ulp_ctl(dev, ulp_type, true, DRV_ACTIVE);
 603
 604        return 0;
 605
 606}
 607EXPORT_SYMBOL(cnic_register_driver);
 608
 609static int cnic_unregister_device(struct cnic_dev *dev, int ulp_type)
 610{
 611        struct cnic_local *cp = dev->cnic_priv;
 612        int i = 0;
 613
 614        if (ulp_type < 0 || ulp_type >= MAX_CNIC_ULP_TYPE) {
 615                pr_err("%s: Bad type %d\n", __func__, ulp_type);
 616                return -EINVAL;
 617        }
 618
 619        if (ulp_type == CNIC_ULP_ISCSI)
 620                cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL);
 621
 622        mutex_lock(&cnic_lock);
 623        if (rcu_access_pointer(cp->ulp_ops[ulp_type])) {
 624                RCU_INIT_POINTER(cp->ulp_ops[ulp_type], NULL);
 625                cnic_put(dev);
 626        } else {
 627                pr_err("%s: device not registered to this ulp type %d\n",
 628                       __func__, ulp_type);
 629                mutex_unlock(&cnic_lock);
 630                return -EINVAL;
 631        }
 632        mutex_unlock(&cnic_lock);
 633
 634        if (ulp_type == CNIC_ULP_FCOE)
 635                dev->fcoe_cap = NULL;
 636
 637        synchronize_rcu();
 638
 639        while (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]) &&
 640               i < 20) {
 641                msleep(100);
 642                i++;
 643        }
 644        if (test_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[ulp_type]))
 645                netdev_warn(dev->netdev, "Failed waiting for ULP up call to complete\n");
 646
 647        if (test_bit(ULP_F_INIT, &cp->ulp_flags[ulp_type]))
 648                cnic_ulp_ctl(dev, ulp_type, false, DRV_UNLOADED);
 649        else
 650                cnic_ulp_ctl(dev, ulp_type, false, DRV_INACTIVE);
 651
 652        return 0;
 653}
 654EXPORT_SYMBOL(cnic_unregister_driver);
 655
 656static int cnic_init_id_tbl(struct cnic_id_tbl *id_tbl, u32 size, u32 start_id,
 657                            u32 next)
 658{
 659        id_tbl->start = start_id;
 660        id_tbl->max = size;
 661        id_tbl->next = next;
 662        spin_lock_init(&id_tbl->lock);
 663        id_tbl->table = kzalloc(DIV_ROUND_UP(size, 32) * 4, GFP_KERNEL);
 664        if (!id_tbl->table)
 665                return -ENOMEM;
 666
 667        return 0;
 668}
 669
 670static void cnic_free_id_tbl(struct cnic_id_tbl *id_tbl)
 671{
 672        kfree(id_tbl->table);
 673        id_tbl->table = NULL;
 674}
 675
 676static int cnic_alloc_id(struct cnic_id_tbl *id_tbl, u32 id)
 677{
 678        int ret = -1;
 679
 680        id -= id_tbl->start;
 681        if (id >= id_tbl->max)
 682                return ret;
 683
 684        spin_lock(&id_tbl->lock);
 685        if (!test_bit(id, id_tbl->table)) {
 686                set_bit(id, id_tbl->table);
 687                ret = 0;
 688        }
 689        spin_unlock(&id_tbl->lock);
 690        return ret;
 691}
 692
 693/* Returns -1 if not successful */
 694static u32 cnic_alloc_new_id(struct cnic_id_tbl *id_tbl)
 695{
 696        u32 id;
 697
 698        spin_lock(&id_tbl->lock);
 699        id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next);
 700        if (id >= id_tbl->max) {
 701                id = -1;
 702                if (id_tbl->next != 0) {
 703                        id = find_first_zero_bit(id_tbl->table, id_tbl->next);
 704                        if (id >= id_tbl->next)
 705                                id = -1;
 706                }
 707        }
 708
 709        if (id < id_tbl->max) {
 710                set_bit(id, id_tbl->table);
 711                id_tbl->next = (id + 1) & (id_tbl->max - 1);
 712                id += id_tbl->start;
 713        }
 714
 715        spin_unlock(&id_tbl->lock);
 716
 717        return id;
 718}
 719
 720static void cnic_free_id(struct cnic_id_tbl *id_tbl, u32 id)
 721{
 722        if (id == -1)
 723                return;
 724
 725        id -= id_tbl->start;
 726        if (id >= id_tbl->max)
 727                return;
 728
 729        clear_bit(id, id_tbl->table);
 730}
 731
 732static void cnic_free_dma(struct cnic_dev *dev, struct cnic_dma *dma)
 733{
 734        int i;
 735
 736        if (!dma->pg_arr)
 737                return;
 738
 739        for (i = 0; i < dma->num_pages; i++) {
 740                if (dma->pg_arr[i]) {
 741                        dma_free_coherent(&dev->pcidev->dev, CNIC_PAGE_SIZE,
 742                                          dma->pg_arr[i], dma->pg_map_arr[i]);
 743                        dma->pg_arr[i] = NULL;
 744                }
 745        }
 746        if (dma->pgtbl) {
 747                dma_free_coherent(&dev->pcidev->dev, dma->pgtbl_size,
 748                                  dma->pgtbl, dma->pgtbl_map);
 749                dma->pgtbl = NULL;
 750        }
 751        kfree(dma->pg_arr);
 752        dma->pg_arr = NULL;
 753        dma->num_pages = 0;
 754}
 755
 756static void cnic_setup_page_tbl(struct cnic_dev *dev, struct cnic_dma *dma)
 757{
 758        int i;
 759        __le32 *page_table = (__le32 *) dma->pgtbl;
 760
 761        for (i = 0; i < dma->num_pages; i++) {
 762                /* Each entry needs to be in big endian format. */
 763                *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32);
 764                page_table++;
 765                *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff);
 766                page_table++;
 767        }
 768}
 769
 770static void cnic_setup_page_tbl_le(struct cnic_dev *dev, struct cnic_dma *dma)
 771{
 772        int i;
 773        __le32 *page_table = (__le32 *) dma->pgtbl;
 774
 775        for (i = 0; i < dma->num_pages; i++) {
 776                /* Each entry needs to be in little endian format. */
 777                *page_table = cpu_to_le32(dma->pg_map_arr[i] & 0xffffffff);
 778                page_table++;
 779                *page_table = cpu_to_le32((u64) dma->pg_map_arr[i] >> 32);
 780                page_table++;
 781        }
 782}
 783
 784static int cnic_alloc_dma(struct cnic_dev *dev, struct cnic_dma *dma,
 785                          int pages, int use_pg_tbl)
 786{
 787        int i, size;
 788        struct cnic_local *cp = dev->cnic_priv;
 789
 790        size = pages * (sizeof(void *) + sizeof(dma_addr_t));
 791        dma->pg_arr = kzalloc(size, GFP_ATOMIC);
 792        if (dma->pg_arr == NULL)
 793                return -ENOMEM;
 794
 795        dma->pg_map_arr = (dma_addr_t *) (dma->pg_arr + pages);
 796        dma->num_pages = pages;
 797
 798        for (i = 0; i < pages; i++) {
 799                dma->pg_arr[i] = dma_alloc_coherent(&dev->pcidev->dev,
 800                                                    CNIC_PAGE_SIZE,
 801                                                    &dma->pg_map_arr[i],
 802                                                    GFP_ATOMIC);
 803                if (dma->pg_arr[i] == NULL)
 804                        goto error;
 805        }
 806        if (!use_pg_tbl)
 807                return 0;
 808
 809        dma->pgtbl_size = ((pages * 8) + CNIC_PAGE_SIZE - 1) &
 810                          ~(CNIC_PAGE_SIZE - 1);
 811        dma->pgtbl = dma_alloc_coherent(&dev->pcidev->dev, dma->pgtbl_size,
 812                                        &dma->pgtbl_map, GFP_ATOMIC);
 813        if (dma->pgtbl == NULL)
 814                goto error;
 815
 816        cp->setup_pgtbl(dev, dma);
 817
 818        return 0;
 819
 820error:
 821        cnic_free_dma(dev, dma);
 822        return -ENOMEM;
 823}
 824
 825static void cnic_free_context(struct cnic_dev *dev)
 826{
 827        struct cnic_local *cp = dev->cnic_priv;
 828        int i;
 829
 830        for (i = 0; i < cp->ctx_blks; i++) {
 831                if (cp->ctx_arr[i].ctx) {
 832                        dma_free_coherent(&dev->pcidev->dev, cp->ctx_blk_size,
 833                                          cp->ctx_arr[i].ctx,
 834                                          cp->ctx_arr[i].mapping);
 835                        cp->ctx_arr[i].ctx = NULL;
 836                }
 837        }
 838}
 839
 840static void __cnic_free_uio_rings(struct cnic_uio_dev *udev)
 841{
 842        if (udev->l2_buf) {
 843                dma_free_coherent(&udev->pdev->dev, udev->l2_buf_size,
 844                                  udev->l2_buf, udev->l2_buf_map);
 845                udev->l2_buf = NULL;
 846        }
 847
 848        if (udev->l2_ring) {
 849                dma_free_coherent(&udev->pdev->dev, udev->l2_ring_size,
 850                                  udev->l2_ring, udev->l2_ring_map);
 851                udev->l2_ring = NULL;
 852        }
 853
 854}
 855
 856static void __cnic_free_uio(struct cnic_uio_dev *udev)
 857{
 858        uio_unregister_device(&udev->cnic_uinfo);
 859
 860        __cnic_free_uio_rings(udev);
 861
 862        pci_dev_put(udev->pdev);
 863        kfree(udev);
 864}
 865
 866static void cnic_free_uio(struct cnic_uio_dev *udev)
 867{
 868        if (!udev)
 869                return;
 870
 871        write_lock(&cnic_dev_lock);
 872        list_del_init(&udev->list);
 873        write_unlock(&cnic_dev_lock);
 874        __cnic_free_uio(udev);
 875}
 876
 877static void cnic_free_resc(struct cnic_dev *dev)
 878{
 879        struct cnic_local *cp = dev->cnic_priv;
 880        struct cnic_uio_dev *udev = cp->udev;
 881
 882        if (udev) {
 883                udev->dev = NULL;
 884                cp->udev = NULL;
 885                if (udev->uio_dev == -1)
 886                        __cnic_free_uio_rings(udev);
 887        }
 888
 889        cnic_free_context(dev);
 890        kfree(cp->ctx_arr);
 891        cp->ctx_arr = NULL;
 892        cp->ctx_blks = 0;
 893
 894        cnic_free_dma(dev, &cp->gbl_buf_info);
 895        cnic_free_dma(dev, &cp->kwq_info);
 896        cnic_free_dma(dev, &cp->kwq_16_data_info);
 897        cnic_free_dma(dev, &cp->kcq2.dma);
 898        cnic_free_dma(dev, &cp->kcq1.dma);
 899        kfree(cp->iscsi_tbl);
 900        cp->iscsi_tbl = NULL;
 901        kfree(cp->ctx_tbl);
 902        cp->ctx_tbl = NULL;
 903
 904        cnic_free_id_tbl(&cp->fcoe_cid_tbl);
 905        cnic_free_id_tbl(&cp->cid_tbl);
 906}
 907
 908static int cnic_alloc_context(struct cnic_dev *dev)
 909{
 910        struct cnic_local *cp = dev->cnic_priv;
 911
 912        if (BNX2_CHIP(cp) == BNX2_CHIP_5709) {
 913                int i, k, arr_size;
 914
 915                cp->ctx_blk_size = CNIC_PAGE_SIZE;
 916                cp->cids_per_blk = CNIC_PAGE_SIZE / 128;
 917                arr_size = BNX2_MAX_CID / cp->cids_per_blk *
 918                           sizeof(struct cnic_ctx);
 919                cp->ctx_arr = kzalloc(arr_size, GFP_KERNEL);
 920                if (cp->ctx_arr == NULL)
 921                        return -ENOMEM;
 922
 923                k = 0;
 924                for (i = 0; i < 2; i++) {
 925                        u32 j, reg, off, lo, hi;
 926
 927                        if (i == 0)
 928                                off = BNX2_PG_CTX_MAP;
 929                        else
 930                                off = BNX2_ISCSI_CTX_MAP;
 931
 932                        reg = cnic_reg_rd_ind(dev, off);
 933                        lo = reg >> 16;
 934                        hi = reg & 0xffff;
 935                        for (j = lo; j < hi; j += cp->cids_per_blk, k++)
 936                                cp->ctx_arr[k].cid = j;
 937                }
 938
 939                cp->ctx_blks = k;
 940                if (cp->ctx_blks >= (BNX2_MAX_CID / cp->cids_per_blk)) {
 941                        cp->ctx_blks = 0;
 942                        return -ENOMEM;
 943                }
 944
 945                for (i = 0; i < cp->ctx_blks; i++) {
 946                        cp->ctx_arr[i].ctx =
 947                                dma_alloc_coherent(&dev->pcidev->dev,
 948                                                   CNIC_PAGE_SIZE,
 949                                                   &cp->ctx_arr[i].mapping,
 950                                                   GFP_KERNEL);
 951                        if (cp->ctx_arr[i].ctx == NULL)
 952                                return -ENOMEM;
 953                }
 954        }
 955        return 0;
 956}
 957
 958static u16 cnic_bnx2_next_idx(u16 idx)
 959{
 960        return idx + 1;
 961}
 962
 963static u16 cnic_bnx2_hw_idx(u16 idx)
 964{
 965        return idx;
 966}
 967
 968static u16 cnic_bnx2x_next_idx(u16 idx)
 969{
 970        idx++;
 971        if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT)
 972                idx++;
 973
 974        return idx;
 975}
 976
 977static u16 cnic_bnx2x_hw_idx(u16 idx)
 978{
 979        if ((idx & MAX_KCQE_CNT) == MAX_KCQE_CNT)
 980                idx++;
 981        return idx;
 982}
 983
 984static int cnic_alloc_kcq(struct cnic_dev *dev, struct kcq_info *info,
 985                          bool use_pg_tbl)
 986{
 987        int err, i, use_page_tbl = 0;
 988        struct kcqe **kcq;
 989
 990        if (use_pg_tbl)
 991                use_page_tbl = 1;
 992
 993        err = cnic_alloc_dma(dev, &info->dma, KCQ_PAGE_CNT, use_page_tbl);
 994        if (err)
 995                return err;
 996
 997        kcq = (struct kcqe **) info->dma.pg_arr;
 998        info->kcq = kcq;
 999
1000        info->next_idx = cnic_bnx2_next_idx;
1001        info->hw_idx = cnic_bnx2_hw_idx;
1002        if (use_pg_tbl)
1003                return 0;
1004
1005        info->next_idx = cnic_bnx2x_next_idx;
1006        info->hw_idx = cnic_bnx2x_hw_idx;
1007
1008        for (i = 0; i < KCQ_PAGE_CNT; i++) {
1009                struct bnx2x_bd_chain_next *next =
1010                        (struct bnx2x_bd_chain_next *) &kcq[i][MAX_KCQE_CNT];
1011                int j = i + 1;
1012
1013                if (j >= KCQ_PAGE_CNT)
1014                        j = 0;
1015                next->addr_hi = (u64) info->dma.pg_map_arr[j] >> 32;
1016                next->addr_lo = info->dma.pg_map_arr[j] & 0xffffffff;
1017        }
1018        return 0;
1019}
1020
1021static int __cnic_alloc_uio_rings(struct cnic_uio_dev *udev, int pages)
1022{
1023        struct cnic_local *cp = udev->dev->cnic_priv;
1024
1025        if (udev->l2_ring)
1026                return 0;
1027
1028        udev->l2_ring_size = pages * CNIC_PAGE_SIZE;
1029        udev->l2_ring = dma_alloc_coherent(&udev->pdev->dev, udev->l2_ring_size,
1030                                           &udev->l2_ring_map,
1031                                           GFP_KERNEL | __GFP_COMP);
1032        if (!udev->l2_ring)
1033                return -ENOMEM;
1034
1035        udev->l2_buf_size = (cp->l2_rx_ring_size + 1) * cp->l2_single_buf_size;
1036        udev->l2_buf_size = CNIC_PAGE_ALIGN(udev->l2_buf_size);
1037        udev->l2_buf = dma_alloc_coherent(&udev->pdev->dev, udev->l2_buf_size,
1038                                          &udev->l2_buf_map,
1039                                          GFP_KERNEL | __GFP_COMP);
1040        if (!udev->l2_buf) {
1041                __cnic_free_uio_rings(udev);
1042                return -ENOMEM;
1043        }
1044
1045        return 0;
1046
1047}
1048
1049static int cnic_alloc_uio_rings(struct cnic_dev *dev, int pages)
1050{
1051        struct cnic_local *cp = dev->cnic_priv;
1052        struct cnic_uio_dev *udev;
1053
1054        list_for_each_entry(udev, &cnic_udev_list, list) {
1055                if (udev->pdev == dev->pcidev) {
1056                        udev->dev = dev;
1057                        if (__cnic_alloc_uio_rings(udev, pages)) {
1058                                udev->dev = NULL;
1059                                return -ENOMEM;
1060                        }
1061                        cp->udev = udev;
1062                        return 0;
1063                }
1064        }
1065
1066        udev = kzalloc(sizeof(struct cnic_uio_dev), GFP_ATOMIC);
1067        if (!udev)
1068                return -ENOMEM;
1069
1070        udev->uio_dev = -1;
1071
1072        udev->dev = dev;
1073        udev->pdev = dev->pcidev;
1074
1075        if (__cnic_alloc_uio_rings(udev, pages))
1076                goto err_udev;
1077
1078        list_add(&udev->list, &cnic_udev_list);
1079
1080        pci_dev_get(udev->pdev);
1081
1082        cp->udev = udev;
1083
1084        return 0;
1085
1086 err_udev:
1087        kfree(udev);
1088        return -ENOMEM;
1089}
1090
1091static int cnic_init_uio(struct cnic_dev *dev)
1092{
1093        struct cnic_local *cp = dev->cnic_priv;
1094        struct cnic_uio_dev *udev = cp->udev;
1095        struct uio_info *uinfo;
1096        int ret = 0;
1097
1098        if (!udev)
1099                return -ENOMEM;
1100
1101        uinfo = &udev->cnic_uinfo;
1102
1103        uinfo->mem[0].addr = pci_resource_start(dev->pcidev, 0);
1104        uinfo->mem[0].internal_addr = dev->regview;
1105        uinfo->mem[0].memtype = UIO_MEM_PHYS;
1106
1107        if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) {
1108                uinfo->mem[0].size = MB_GET_CID_ADDR(TX_TSS_CID +
1109                                                     TX_MAX_TSS_RINGS + 1);
1110                uinfo->mem[1].addr = (unsigned long) cp->status_blk.gen &
1111                                        CNIC_PAGE_MASK;
1112                if (cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)
1113                        uinfo->mem[1].size = BNX2_SBLK_MSIX_ALIGN_SIZE * 9;
1114                else
1115                        uinfo->mem[1].size = BNX2_SBLK_MSIX_ALIGN_SIZE;
1116
1117                uinfo->name = "bnx2_cnic";
1118        } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) {
1119                uinfo->mem[0].size = pci_resource_len(dev->pcidev, 0);
1120
1121                uinfo->mem[1].addr = (unsigned long) cp->bnx2x_def_status_blk &
1122                        CNIC_PAGE_MASK;
1123                uinfo->mem[1].size = sizeof(*cp->bnx2x_def_status_blk);
1124
1125                uinfo->name = "bnx2x_cnic";
1126        }
1127
1128        uinfo->mem[1].memtype = UIO_MEM_LOGICAL;
1129
1130        uinfo->mem[2].addr = (unsigned long) udev->l2_ring;
1131        uinfo->mem[2].size = udev->l2_ring_size;
1132        uinfo->mem[2].memtype = UIO_MEM_LOGICAL;
1133
1134        uinfo->mem[3].addr = (unsigned long) udev->l2_buf;
1135        uinfo->mem[3].size = udev->l2_buf_size;
1136        uinfo->mem[3].memtype = UIO_MEM_LOGICAL;
1137
1138        uinfo->version = CNIC_MODULE_VERSION;
1139        uinfo->irq = UIO_IRQ_CUSTOM;
1140
1141        uinfo->open = cnic_uio_open;
1142        uinfo->release = cnic_uio_close;
1143
1144        if (udev->uio_dev == -1) {
1145                if (!uinfo->priv) {
1146                        uinfo->priv = udev;
1147
1148                        ret = uio_register_device(&udev->pdev->dev, uinfo);
1149                }
1150        } else {
1151                cnic_init_rings(dev);
1152        }
1153
1154        return ret;
1155}
1156
1157static int cnic_alloc_bnx2_resc(struct cnic_dev *dev)
1158{
1159        struct cnic_local *cp = dev->cnic_priv;
1160        int ret;
1161
1162        ret = cnic_alloc_dma(dev, &cp->kwq_info, KWQ_PAGE_CNT, 1);
1163        if (ret)
1164                goto error;
1165        cp->kwq = (struct kwqe **) cp->kwq_info.pg_arr;
1166
1167        ret = cnic_alloc_kcq(dev, &cp->kcq1, true);
1168        if (ret)
1169                goto error;
1170
1171        ret = cnic_alloc_context(dev);
1172        if (ret)
1173                goto error;
1174
1175        ret = cnic_alloc_uio_rings(dev, 2);
1176        if (ret)
1177                goto error;
1178
1179        ret = cnic_init_uio(dev);
1180        if (ret)
1181                goto error;
1182
1183        return 0;
1184
1185error:
1186        cnic_free_resc(dev);
1187        return ret;
1188}
1189
1190static int cnic_alloc_bnx2x_context(struct cnic_dev *dev)
1191{
1192        struct cnic_local *cp = dev->cnic_priv;
1193        struct bnx2x *bp = netdev_priv(dev->netdev);
1194        int ctx_blk_size = cp->ethdev->ctx_blk_size;
1195        int total_mem, blks, i;
1196
1197        total_mem = BNX2X_CONTEXT_MEM_SIZE * cp->max_cid_space;
1198        blks = total_mem / ctx_blk_size;
1199        if (total_mem % ctx_blk_size)
1200                blks++;
1201
1202        if (blks > cp->ethdev->ctx_tbl_len)
1203                return -ENOMEM;
1204
1205        cp->ctx_arr = kcalloc(blks, sizeof(struct cnic_ctx), GFP_KERNEL);
1206        if (cp->ctx_arr == NULL)
1207                return -ENOMEM;
1208
1209        cp->ctx_blks = blks;
1210        cp->ctx_blk_size = ctx_blk_size;
1211        if (!CHIP_IS_E1(bp))
1212                cp->ctx_align = 0;
1213        else
1214                cp->ctx_align = ctx_blk_size;
1215
1216        cp->cids_per_blk = ctx_blk_size / BNX2X_CONTEXT_MEM_SIZE;
1217
1218        for (i = 0; i < blks; i++) {
1219                cp->ctx_arr[i].ctx =
1220                        dma_alloc_coherent(&dev->pcidev->dev, cp->ctx_blk_size,
1221                                           &cp->ctx_arr[i].mapping,
1222                                           GFP_KERNEL);
1223                if (cp->ctx_arr[i].ctx == NULL)
1224                        return -ENOMEM;
1225
1226                if (cp->ctx_align && cp->ctx_blk_size == ctx_blk_size) {
1227                        if (cp->ctx_arr[i].mapping & (cp->ctx_align - 1)) {
1228                                cnic_free_context(dev);
1229                                cp->ctx_blk_size += cp->ctx_align;
1230                                i = -1;
1231                                continue;
1232                        }
1233                }
1234        }
1235        return 0;
1236}
1237
1238static int cnic_alloc_bnx2x_resc(struct cnic_dev *dev)
1239{
1240        struct cnic_local *cp = dev->cnic_priv;
1241        struct bnx2x *bp = netdev_priv(dev->netdev);
1242        struct cnic_eth_dev *ethdev = cp->ethdev;
1243        u32 start_cid = ethdev->starting_cid;
1244        int i, j, n, ret, pages;
1245        struct cnic_dma *kwq_16_dma = &cp->kwq_16_data_info;
1246
1247        cp->max_cid_space = MAX_ISCSI_TBL_SZ;
1248        cp->iscsi_start_cid = start_cid;
1249        cp->fcoe_start_cid = start_cid + MAX_ISCSI_TBL_SZ;
1250
1251        if (BNX2X_CHIP_IS_E2_PLUS(bp)) {
1252                cp->max_cid_space += dev->max_fcoe_conn;
1253                cp->fcoe_init_cid = ethdev->fcoe_init_cid;
1254                if (!cp->fcoe_init_cid)
1255                        cp->fcoe_init_cid = 0x10;
1256        }
1257
1258        cp->iscsi_tbl = kzalloc(sizeof(struct cnic_iscsi) * MAX_ISCSI_TBL_SZ,
1259                                GFP_KERNEL);
1260        if (!cp->iscsi_tbl)
1261                goto error;
1262
1263        cp->ctx_tbl = kzalloc(sizeof(struct cnic_context) *
1264                                cp->max_cid_space, GFP_KERNEL);
1265        if (!cp->ctx_tbl)
1266                goto error;
1267
1268        for (i = 0; i < MAX_ISCSI_TBL_SZ; i++) {
1269                cp->ctx_tbl[i].proto.iscsi = &cp->iscsi_tbl[i];
1270                cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_ISCSI;
1271        }
1272
1273        for (i = MAX_ISCSI_TBL_SZ; i < cp->max_cid_space; i++)
1274                cp->ctx_tbl[i].ulp_proto_id = CNIC_ULP_FCOE;
1275
1276        pages = CNIC_PAGE_ALIGN(cp->max_cid_space * CNIC_KWQ16_DATA_SIZE) /
1277                CNIC_PAGE_SIZE;
1278
1279        ret = cnic_alloc_dma(dev, kwq_16_dma, pages, 0);
1280        if (ret)
1281                return -ENOMEM;
1282
1283        n = CNIC_PAGE_SIZE / CNIC_KWQ16_DATA_SIZE;
1284        for (i = 0, j = 0; i < cp->max_cid_space; i++) {
1285                long off = CNIC_KWQ16_DATA_SIZE * (i % n);
1286
1287                cp->ctx_tbl[i].kwqe_data = kwq_16_dma->pg_arr[j] + off;
1288                cp->ctx_tbl[i].kwqe_data_mapping = kwq_16_dma->pg_map_arr[j] +
1289                                                   off;
1290
1291                if ((i % n) == (n - 1))
1292                        j++;
1293        }
1294
1295        ret = cnic_alloc_kcq(dev, &cp->kcq1, false);
1296        if (ret)
1297                goto error;
1298
1299        if (CNIC_SUPPORTS_FCOE(bp)) {
1300                ret = cnic_alloc_kcq(dev, &cp->kcq2, true);
1301                if (ret)
1302                        goto error;
1303        }
1304
1305        pages = CNIC_PAGE_ALIGN(BNX2X_ISCSI_GLB_BUF_SIZE) / CNIC_PAGE_SIZE;
1306        ret = cnic_alloc_dma(dev, &cp->gbl_buf_info, pages, 0);
1307        if (ret)
1308                goto error;
1309
1310        ret = cnic_alloc_bnx2x_context(dev);
1311        if (ret)
1312                goto error;
1313
1314        if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)
1315                return 0;
1316
1317        cp->bnx2x_def_status_blk = cp->ethdev->irq_arr[1].status_blk;
1318
1319        cp->l2_rx_ring_size = 15;
1320
1321        ret = cnic_alloc_uio_rings(dev, 4);
1322        if (ret)
1323                goto error;
1324
1325        ret = cnic_init_uio(dev);
1326        if (ret)
1327                goto error;
1328
1329        return 0;
1330
1331error:
1332        cnic_free_resc(dev);
1333        return -ENOMEM;
1334}
1335
1336static inline u32 cnic_kwq_avail(struct cnic_local *cp)
1337{
1338        return cp->max_kwq_idx -
1339                ((cp->kwq_prod_idx - cp->kwq_con_idx) & cp->max_kwq_idx);
1340}
1341
1342static int cnic_submit_bnx2_kwqes(struct cnic_dev *dev, struct kwqe *wqes[],
1343                                  u32 num_wqes)
1344{
1345        struct cnic_local *cp = dev->cnic_priv;
1346        struct kwqe *prod_qe;
1347        u16 prod, sw_prod, i;
1348
1349        if (!test_bit(CNIC_F_CNIC_UP, &dev->flags))
1350                return -EAGAIN;         /* bnx2 is down */
1351
1352        spin_lock_bh(&cp->cnic_ulp_lock);
1353        if (num_wqes > cnic_kwq_avail(cp) &&
1354            !test_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags)) {
1355                spin_unlock_bh(&cp->cnic_ulp_lock);
1356                return -EAGAIN;
1357        }
1358
1359        clear_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags);
1360
1361        prod = cp->kwq_prod_idx;
1362        sw_prod = prod & MAX_KWQ_IDX;
1363        for (i = 0; i < num_wqes; i++) {
1364                prod_qe = &cp->kwq[KWQ_PG(sw_prod)][KWQ_IDX(sw_prod)];
1365                memcpy(prod_qe, wqes[i], sizeof(struct kwqe));
1366                prod++;
1367                sw_prod = prod & MAX_KWQ_IDX;
1368        }
1369        cp->kwq_prod_idx = prod;
1370
1371        CNIC_WR16(dev, cp->kwq_io_addr, cp->kwq_prod_idx);
1372
1373        spin_unlock_bh(&cp->cnic_ulp_lock);
1374        return 0;
1375}
1376
1377static void *cnic_get_kwqe_16_data(struct cnic_local *cp, u32 l5_cid,
1378                                   union l5cm_specific_data *l5_data)
1379{
1380        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
1381        dma_addr_t map;
1382
1383        map = ctx->kwqe_data_mapping;
1384        l5_data->phy_address.lo = (u64) map & 0xffffffff;
1385        l5_data->phy_address.hi = (u64) map >> 32;
1386        return ctx->kwqe_data;
1387}
1388
1389static int cnic_submit_kwqe_16(struct cnic_dev *dev, u32 cmd, u32 cid,
1390                                u32 type, union l5cm_specific_data *l5_data)
1391{
1392        struct cnic_local *cp = dev->cnic_priv;
1393        struct bnx2x *bp = netdev_priv(dev->netdev);
1394        struct l5cm_spe kwqe;
1395        struct kwqe_16 *kwq[1];
1396        u16 type_16;
1397        int ret;
1398
1399        kwqe.hdr.conn_and_cmd_data =
1400                cpu_to_le32(((cmd << SPE_HDR_CMD_ID_SHIFT) |
1401                             BNX2X_HW_CID(bp, cid)));
1402
1403        type_16 = (type << SPE_HDR_CONN_TYPE_SHIFT) & SPE_HDR_CONN_TYPE;
1404        type_16 |= (bp->pfid << SPE_HDR_FUNCTION_ID_SHIFT) &
1405                   SPE_HDR_FUNCTION_ID;
1406
1407        kwqe.hdr.type = cpu_to_le16(type_16);
1408        kwqe.hdr.reserved1 = 0;
1409        kwqe.data.phy_address.lo = cpu_to_le32(l5_data->phy_address.lo);
1410        kwqe.data.phy_address.hi = cpu_to_le32(l5_data->phy_address.hi);
1411
1412        kwq[0] = (struct kwqe_16 *) &kwqe;
1413
1414        spin_lock_bh(&cp->cnic_ulp_lock);
1415        ret = cp->ethdev->drv_submit_kwqes_16(dev->netdev, kwq, 1);
1416        spin_unlock_bh(&cp->cnic_ulp_lock);
1417
1418        if (ret == 1)
1419                return 0;
1420
1421        return ret;
1422}
1423
1424static void cnic_reply_bnx2x_kcqes(struct cnic_dev *dev, int ulp_type,
1425                                   struct kcqe *cqes[], u32 num_cqes)
1426{
1427        struct cnic_local *cp = dev->cnic_priv;
1428        struct cnic_ulp_ops *ulp_ops;
1429
1430        rcu_read_lock();
1431        ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]);
1432        if (likely(ulp_ops)) {
1433                ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type],
1434                                          cqes, num_cqes);
1435        }
1436        rcu_read_unlock();
1437}
1438
1439static void cnic_bnx2x_set_tcp_options(struct cnic_dev *dev, int time_stamps,
1440                                       int en_tcp_dack)
1441{
1442        struct bnx2x *bp = netdev_priv(dev->netdev);
1443        u8 xstorm_flags = XSTORM_L5CM_TCP_FLAGS_WND_SCL_EN;
1444        u16 tstorm_flags = 0;
1445
1446        if (time_stamps) {
1447                xstorm_flags |= XSTORM_L5CM_TCP_FLAGS_TS_ENABLED;
1448                tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_TS_ENABLED;
1449        }
1450        if (en_tcp_dack)
1451                tstorm_flags |= TSTORM_L5CM_TCP_FLAGS_DELAYED_ACK_EN;
1452
1453        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
1454                 XSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), xstorm_flags);
1455
1456        CNIC_WR16(dev, BAR_TSTRORM_INTMEM +
1457                  TSTORM_ISCSI_TCP_VARS_FLAGS_OFFSET(bp->pfid), tstorm_flags);
1458}
1459
1460static int cnic_bnx2x_iscsi_init1(struct cnic_dev *dev, struct kwqe *kwqe)
1461{
1462        struct cnic_local *cp = dev->cnic_priv;
1463        struct bnx2x *bp = netdev_priv(dev->netdev);
1464        struct iscsi_kwqe_init1 *req1 = (struct iscsi_kwqe_init1 *) kwqe;
1465        int hq_bds, pages;
1466        u32 pfid = bp->pfid;
1467
1468        cp->num_iscsi_tasks = req1->num_tasks_per_conn;
1469        cp->num_ccells = req1->num_ccells_per_conn;
1470        cp->task_array_size = BNX2X_ISCSI_TASK_CONTEXT_SIZE *
1471                              cp->num_iscsi_tasks;
1472        cp->r2tq_size = cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS *
1473                        BNX2X_ISCSI_R2TQE_SIZE;
1474        cp->hq_size = cp->num_ccells * BNX2X_ISCSI_HQ_BD_SIZE;
1475        pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE;
1476        hq_bds = pages * (CNIC_PAGE_SIZE / BNX2X_ISCSI_HQ_BD_SIZE);
1477        cp->num_cqs = req1->num_cqs;
1478
1479        if (!dev->max_iscsi_conn)
1480                return 0;
1481
1482        /* init Tstorm RAM */
1483        CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_RQ_SIZE_OFFSET(pfid),
1484                  req1->rq_num_wqes);
1485        CNIC_WR16(dev, BAR_TSTRORM_INTMEM + TSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid),
1486                  CNIC_PAGE_SIZE);
1487        CNIC_WR8(dev, BAR_TSTRORM_INTMEM +
1488                 TSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS);
1489        CNIC_WR16(dev, BAR_TSTRORM_INTMEM +
1490                  TSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid),
1491                  req1->num_tasks_per_conn);
1492
1493        /* init Ustorm RAM */
1494        CNIC_WR16(dev, BAR_USTRORM_INTMEM +
1495                  USTORM_ISCSI_RQ_BUFFER_SIZE_OFFSET(pfid),
1496                  req1->rq_buffer_size);
1497        CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_PAGE_SIZE_OFFSET(pfid),
1498                  CNIC_PAGE_SIZE);
1499        CNIC_WR8(dev, BAR_USTRORM_INTMEM +
1500                 USTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS);
1501        CNIC_WR16(dev, BAR_USTRORM_INTMEM +
1502                  USTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid),
1503                  req1->num_tasks_per_conn);
1504        CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_RQ_SIZE_OFFSET(pfid),
1505                  req1->rq_num_wqes);
1506        CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_CQ_SIZE_OFFSET(pfid),
1507                  req1->cq_num_wqes);
1508        CNIC_WR16(dev, BAR_USTRORM_INTMEM + USTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid),
1509                  cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS);
1510
1511        /* init Xstorm RAM */
1512        CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid),
1513                  CNIC_PAGE_SIZE);
1514        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
1515                 XSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS);
1516        CNIC_WR16(dev, BAR_XSTRORM_INTMEM +
1517                  XSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid),
1518                  req1->num_tasks_per_conn);
1519        CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_HQ_SIZE_OFFSET(pfid),
1520                  hq_bds);
1521        CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_SQ_SIZE_OFFSET(pfid),
1522                  req1->num_tasks_per_conn);
1523        CNIC_WR16(dev, BAR_XSTRORM_INTMEM + XSTORM_ISCSI_R2TQ_SIZE_OFFSET(pfid),
1524                  cp->num_iscsi_tasks * BNX2X_ISCSI_MAX_PENDING_R2TS);
1525
1526        /* init Cstorm RAM */
1527        CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_PAGE_SIZE_OFFSET(pfid),
1528                  CNIC_PAGE_SIZE);
1529        CNIC_WR8(dev, BAR_CSTRORM_INTMEM +
1530                 CSTORM_ISCSI_PAGE_SIZE_LOG_OFFSET(pfid), CNIC_PAGE_BITS);
1531        CNIC_WR16(dev, BAR_CSTRORM_INTMEM +
1532                  CSTORM_ISCSI_NUM_OF_TASKS_OFFSET(pfid),
1533                  req1->num_tasks_per_conn);
1534        CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_CQ_SIZE_OFFSET(pfid),
1535                  req1->cq_num_wqes);
1536        CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_ISCSI_HQ_SIZE_OFFSET(pfid),
1537                  hq_bds);
1538
1539        cnic_bnx2x_set_tcp_options(dev,
1540                        req1->flags & ISCSI_KWQE_INIT1_TIME_STAMPS_ENABLE,
1541                        req1->flags & ISCSI_KWQE_INIT1_DELAYED_ACK_ENABLE);
1542
1543        return 0;
1544}
1545
1546static int cnic_bnx2x_iscsi_init2(struct cnic_dev *dev, struct kwqe *kwqe)
1547{
1548        struct iscsi_kwqe_init2 *req2 = (struct iscsi_kwqe_init2 *) kwqe;
1549        struct bnx2x *bp = netdev_priv(dev->netdev);
1550        u32 pfid = bp->pfid;
1551        struct iscsi_kcqe kcqe;
1552        struct kcqe *cqes[1];
1553
1554        memset(&kcqe, 0, sizeof(kcqe));
1555        if (!dev->max_iscsi_conn) {
1556                kcqe.completion_status =
1557                        ISCSI_KCQE_COMPLETION_STATUS_ISCSI_NOT_SUPPORTED;
1558                goto done;
1559        }
1560
1561        CNIC_WR(dev, BAR_TSTRORM_INTMEM +
1562                TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]);
1563        CNIC_WR(dev, BAR_TSTRORM_INTMEM +
1564                TSTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4,
1565                req2->error_bit_map[1]);
1566
1567        CNIC_WR16(dev, BAR_USTRORM_INTMEM +
1568                  USTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn);
1569        CNIC_WR(dev, BAR_USTRORM_INTMEM +
1570                USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid), req2->error_bit_map[0]);
1571        CNIC_WR(dev, BAR_USTRORM_INTMEM +
1572                USTORM_ISCSI_ERROR_BITMAP_OFFSET(pfid) + 4,
1573                req2->error_bit_map[1]);
1574
1575        CNIC_WR16(dev, BAR_CSTRORM_INTMEM +
1576                  CSTORM_ISCSI_CQ_SQN_SIZE_OFFSET(pfid), req2->max_cq_sqn);
1577
1578        kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS;
1579
1580done:
1581        kcqe.op_code = ISCSI_KCQE_OPCODE_INIT;
1582        cqes[0] = (struct kcqe *) &kcqe;
1583        cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1);
1584
1585        return 0;
1586}
1587
1588static void cnic_free_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid)
1589{
1590        struct cnic_local *cp = dev->cnic_priv;
1591        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
1592
1593        if (ctx->ulp_proto_id == CNIC_ULP_ISCSI) {
1594                struct cnic_iscsi *iscsi = ctx->proto.iscsi;
1595
1596                cnic_free_dma(dev, &iscsi->hq_info);
1597                cnic_free_dma(dev, &iscsi->r2tq_info);
1598                cnic_free_dma(dev, &iscsi->task_array_info);
1599                cnic_free_id(&cp->cid_tbl, ctx->cid);
1600        } else {
1601                cnic_free_id(&cp->fcoe_cid_tbl, ctx->cid);
1602        }
1603
1604        ctx->cid = 0;
1605}
1606
1607static int cnic_alloc_bnx2x_conn_resc(struct cnic_dev *dev, u32 l5_cid)
1608{
1609        u32 cid;
1610        int ret, pages;
1611        struct cnic_local *cp = dev->cnic_priv;
1612        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
1613        struct cnic_iscsi *iscsi = ctx->proto.iscsi;
1614
1615        if (ctx->ulp_proto_id == CNIC_ULP_FCOE) {
1616                cid = cnic_alloc_new_id(&cp->fcoe_cid_tbl);
1617                if (cid == -1) {
1618                        ret = -ENOMEM;
1619                        goto error;
1620                }
1621                ctx->cid = cid;
1622                return 0;
1623        }
1624
1625        cid = cnic_alloc_new_id(&cp->cid_tbl);
1626        if (cid == -1) {
1627                ret = -ENOMEM;
1628                goto error;
1629        }
1630
1631        ctx->cid = cid;
1632        pages = CNIC_PAGE_ALIGN(cp->task_array_size) / CNIC_PAGE_SIZE;
1633
1634        ret = cnic_alloc_dma(dev, &iscsi->task_array_info, pages, 1);
1635        if (ret)
1636                goto error;
1637
1638        pages = CNIC_PAGE_ALIGN(cp->r2tq_size) / CNIC_PAGE_SIZE;
1639        ret = cnic_alloc_dma(dev, &iscsi->r2tq_info, pages, 1);
1640        if (ret)
1641                goto error;
1642
1643        pages = CNIC_PAGE_ALIGN(cp->hq_size) / CNIC_PAGE_SIZE;
1644        ret = cnic_alloc_dma(dev, &iscsi->hq_info, pages, 1);
1645        if (ret)
1646                goto error;
1647
1648        return 0;
1649
1650error:
1651        cnic_free_bnx2x_conn_resc(dev, l5_cid);
1652        return ret;
1653}
1654
1655static void *cnic_get_bnx2x_ctx(struct cnic_dev *dev, u32 cid, int init,
1656                                struct regpair *ctx_addr)
1657{
1658        struct cnic_local *cp = dev->cnic_priv;
1659        struct cnic_eth_dev *ethdev = cp->ethdev;
1660        int blk = (cid - ethdev->starting_cid) / cp->cids_per_blk;
1661        int off = (cid - ethdev->starting_cid) % cp->cids_per_blk;
1662        unsigned long align_off = 0;
1663        dma_addr_t ctx_map;
1664        void *ctx;
1665
1666        if (cp->ctx_align) {
1667                unsigned long mask = cp->ctx_align - 1;
1668
1669                if (cp->ctx_arr[blk].mapping & mask)
1670                        align_off = cp->ctx_align -
1671                                    (cp->ctx_arr[blk].mapping & mask);
1672        }
1673        ctx_map = cp->ctx_arr[blk].mapping + align_off +
1674                (off * BNX2X_CONTEXT_MEM_SIZE);
1675        ctx = cp->ctx_arr[blk].ctx + align_off +
1676              (off * BNX2X_CONTEXT_MEM_SIZE);
1677        if (init)
1678                memset(ctx, 0, BNX2X_CONTEXT_MEM_SIZE);
1679
1680        ctx_addr->lo = ctx_map & 0xffffffff;
1681        ctx_addr->hi = (u64) ctx_map >> 32;
1682        return ctx;
1683}
1684
1685static int cnic_setup_bnx2x_ctx(struct cnic_dev *dev, struct kwqe *wqes[],
1686                                u32 num)
1687{
1688        struct cnic_local *cp = dev->cnic_priv;
1689        struct bnx2x *bp = netdev_priv(dev->netdev);
1690        struct iscsi_kwqe_conn_offload1 *req1 =
1691                        (struct iscsi_kwqe_conn_offload1 *) wqes[0];
1692        struct iscsi_kwqe_conn_offload2 *req2 =
1693                        (struct iscsi_kwqe_conn_offload2 *) wqes[1];
1694        struct iscsi_kwqe_conn_offload3 *req3;
1695        struct cnic_context *ctx = &cp->ctx_tbl[req1->iscsi_conn_id];
1696        struct cnic_iscsi *iscsi = ctx->proto.iscsi;
1697        u32 cid = ctx->cid;
1698        u32 hw_cid = BNX2X_HW_CID(bp, cid);
1699        struct iscsi_context *ictx;
1700        struct regpair context_addr;
1701        int i, j, n = 2, n_max;
1702        u8 port = BP_PORT(bp);
1703
1704        ctx->ctx_flags = 0;
1705        if (!req2->num_additional_wqes)
1706                return -EINVAL;
1707
1708        n_max = req2->num_additional_wqes + 2;
1709
1710        ictx = cnic_get_bnx2x_ctx(dev, cid, 1, &context_addr);
1711        if (ictx == NULL)
1712                return -ENOMEM;
1713
1714        req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++];
1715
1716        ictx->xstorm_ag_context.hq_prod = 1;
1717
1718        ictx->xstorm_st_context.iscsi.first_burst_length =
1719                ISCSI_DEF_FIRST_BURST_LEN;
1720        ictx->xstorm_st_context.iscsi.max_send_pdu_length =
1721                ISCSI_DEF_MAX_RECV_SEG_LEN;
1722        ictx->xstorm_st_context.iscsi.sq_pbl_base.lo =
1723                req1->sq_page_table_addr_lo;
1724        ictx->xstorm_st_context.iscsi.sq_pbl_base.hi =
1725                req1->sq_page_table_addr_hi;
1726        ictx->xstorm_st_context.iscsi.sq_curr_pbe.lo = req2->sq_first_pte.hi;
1727        ictx->xstorm_st_context.iscsi.sq_curr_pbe.hi = req2->sq_first_pte.lo;
1728        ictx->xstorm_st_context.iscsi.hq_pbl_base.lo =
1729                iscsi->hq_info.pgtbl_map & 0xffffffff;
1730        ictx->xstorm_st_context.iscsi.hq_pbl_base.hi =
1731                (u64) iscsi->hq_info.pgtbl_map >> 32;
1732        ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.lo =
1733                iscsi->hq_info.pgtbl[0];
1734        ictx->xstorm_st_context.iscsi.hq_curr_pbe_base.hi =
1735                iscsi->hq_info.pgtbl[1];
1736        ictx->xstorm_st_context.iscsi.r2tq_pbl_base.lo =
1737                iscsi->r2tq_info.pgtbl_map & 0xffffffff;
1738        ictx->xstorm_st_context.iscsi.r2tq_pbl_base.hi =
1739                (u64) iscsi->r2tq_info.pgtbl_map >> 32;
1740        ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.lo =
1741                iscsi->r2tq_info.pgtbl[0];
1742        ictx->xstorm_st_context.iscsi.r2tq_curr_pbe_base.hi =
1743                iscsi->r2tq_info.pgtbl[1];
1744        ictx->xstorm_st_context.iscsi.task_pbl_base.lo =
1745                iscsi->task_array_info.pgtbl_map & 0xffffffff;
1746        ictx->xstorm_st_context.iscsi.task_pbl_base.hi =
1747                (u64) iscsi->task_array_info.pgtbl_map >> 32;
1748        ictx->xstorm_st_context.iscsi.task_pbl_cache_idx =
1749                BNX2X_ISCSI_PBL_NOT_CACHED;
1750        ictx->xstorm_st_context.iscsi.flags.flags |=
1751                XSTORM_ISCSI_CONTEXT_FLAGS_B_IMMEDIATE_DATA;
1752        ictx->xstorm_st_context.iscsi.flags.flags |=
1753                XSTORM_ISCSI_CONTEXT_FLAGS_B_INITIAL_R2T;
1754        ictx->xstorm_st_context.common.ethernet.reserved_vlan_type =
1755                ETH_P_8021Q;
1756        if (BNX2X_CHIP_IS_E2_PLUS(bp) &&
1757            bp->common.chip_port_mode == CHIP_2_PORT_MODE) {
1758
1759                port = 0;
1760        }
1761        ictx->xstorm_st_context.common.flags =
1762                1 << XSTORM_COMMON_CONTEXT_SECTION_PHYSQ_INITIALIZED_SHIFT;
1763        ictx->xstorm_st_context.common.flags =
1764                port << XSTORM_COMMON_CONTEXT_SECTION_PBF_PORT_SHIFT;
1765
1766        ictx->tstorm_st_context.iscsi.hdr_bytes_2_fetch = ISCSI_HEADER_SIZE;
1767        /* TSTORM requires the base address of RQ DB & not PTE */
1768        ictx->tstorm_st_context.iscsi.rq_db_phy_addr.lo =
1769                req2->rq_page_table_addr_lo & CNIC_PAGE_MASK;
1770        ictx->tstorm_st_context.iscsi.rq_db_phy_addr.hi =
1771                req2->rq_page_table_addr_hi;
1772        ictx->tstorm_st_context.iscsi.iscsi_conn_id = req1->iscsi_conn_id;
1773        ictx->tstorm_st_context.tcp.cwnd = 0x5A8;
1774        ictx->tstorm_st_context.tcp.flags2 |=
1775                TSTORM_TCP_ST_CONTEXT_SECTION_DA_EN;
1776        ictx->tstorm_st_context.tcp.ooo_support_mode =
1777                TCP_TSTORM_OOO_DROP_AND_PROC_ACK;
1778
1779        ictx->timers_context.flags |= TIMERS_BLOCK_CONTEXT_CONN_VALID_FLG;
1780
1781        ictx->ustorm_st_context.ring.rq.pbl_base.lo =
1782                req2->rq_page_table_addr_lo;
1783        ictx->ustorm_st_context.ring.rq.pbl_base.hi =
1784                req2->rq_page_table_addr_hi;
1785        ictx->ustorm_st_context.ring.rq.curr_pbe.lo = req3->qp_first_pte[0].hi;
1786        ictx->ustorm_st_context.ring.rq.curr_pbe.hi = req3->qp_first_pte[0].lo;
1787        ictx->ustorm_st_context.ring.r2tq.pbl_base.lo =
1788                iscsi->r2tq_info.pgtbl_map & 0xffffffff;
1789        ictx->ustorm_st_context.ring.r2tq.pbl_base.hi =
1790                (u64) iscsi->r2tq_info.pgtbl_map >> 32;
1791        ictx->ustorm_st_context.ring.r2tq.curr_pbe.lo =
1792                iscsi->r2tq_info.pgtbl[0];
1793        ictx->ustorm_st_context.ring.r2tq.curr_pbe.hi =
1794                iscsi->r2tq_info.pgtbl[1];
1795        ictx->ustorm_st_context.ring.cq_pbl_base.lo =
1796                req1->cq_page_table_addr_lo;
1797        ictx->ustorm_st_context.ring.cq_pbl_base.hi =
1798                req1->cq_page_table_addr_hi;
1799        ictx->ustorm_st_context.ring.cq[0].cq_sn = ISCSI_INITIAL_SN;
1800        ictx->ustorm_st_context.ring.cq[0].curr_pbe.lo = req2->cq_first_pte.hi;
1801        ictx->ustorm_st_context.ring.cq[0].curr_pbe.hi = req2->cq_first_pte.lo;
1802        ictx->ustorm_st_context.task_pbe_cache_index =
1803                BNX2X_ISCSI_PBL_NOT_CACHED;
1804        ictx->ustorm_st_context.task_pdu_cache_index =
1805                BNX2X_ISCSI_PDU_HEADER_NOT_CACHED;
1806
1807        for (i = 1, j = 1; i < cp->num_cqs; i++, j++) {
1808                if (j == 3) {
1809                        if (n >= n_max)
1810                                break;
1811                        req3 = (struct iscsi_kwqe_conn_offload3 *) wqes[n++];
1812                        j = 0;
1813                }
1814                ictx->ustorm_st_context.ring.cq[i].cq_sn = ISCSI_INITIAL_SN;
1815                ictx->ustorm_st_context.ring.cq[i].curr_pbe.lo =
1816                        req3->qp_first_pte[j].hi;
1817                ictx->ustorm_st_context.ring.cq[i].curr_pbe.hi =
1818                        req3->qp_first_pte[j].lo;
1819        }
1820
1821        ictx->ustorm_st_context.task_pbl_base.lo =
1822                iscsi->task_array_info.pgtbl_map & 0xffffffff;
1823        ictx->ustorm_st_context.task_pbl_base.hi =
1824                (u64) iscsi->task_array_info.pgtbl_map >> 32;
1825        ictx->ustorm_st_context.tce_phy_addr.lo =
1826                iscsi->task_array_info.pgtbl[0];
1827        ictx->ustorm_st_context.tce_phy_addr.hi =
1828                iscsi->task_array_info.pgtbl[1];
1829        ictx->ustorm_st_context.iscsi_conn_id = req1->iscsi_conn_id;
1830        ictx->ustorm_st_context.num_cqs = cp->num_cqs;
1831        ictx->ustorm_st_context.negotiated_rx |= ISCSI_DEF_MAX_RECV_SEG_LEN;
1832        ictx->ustorm_st_context.negotiated_rx_and_flags |=
1833                ISCSI_DEF_MAX_BURST_LEN;
1834        ictx->ustorm_st_context.negotiated_rx |=
1835                ISCSI_DEFAULT_MAX_OUTSTANDING_R2T <<
1836                USTORM_ISCSI_ST_CONTEXT_MAX_OUTSTANDING_R2TS_SHIFT;
1837
1838        ictx->cstorm_st_context.hq_pbl_base.lo =
1839                iscsi->hq_info.pgtbl_map & 0xffffffff;
1840        ictx->cstorm_st_context.hq_pbl_base.hi =
1841                (u64) iscsi->hq_info.pgtbl_map >> 32;
1842        ictx->cstorm_st_context.hq_curr_pbe.lo = iscsi->hq_info.pgtbl[0];
1843        ictx->cstorm_st_context.hq_curr_pbe.hi = iscsi->hq_info.pgtbl[1];
1844        ictx->cstorm_st_context.task_pbl_base.lo =
1845                iscsi->task_array_info.pgtbl_map & 0xffffffff;
1846        ictx->cstorm_st_context.task_pbl_base.hi =
1847                (u64) iscsi->task_array_info.pgtbl_map >> 32;
1848        /* CSTORM and USTORM initialization is different, CSTORM requires
1849         * CQ DB base & not PTE addr */
1850        ictx->cstorm_st_context.cq_db_base.lo =
1851                req1->cq_page_table_addr_lo & CNIC_PAGE_MASK;
1852        ictx->cstorm_st_context.cq_db_base.hi = req1->cq_page_table_addr_hi;
1853        ictx->cstorm_st_context.iscsi_conn_id = req1->iscsi_conn_id;
1854        ictx->cstorm_st_context.cq_proc_en_bit_map = (1 << cp->num_cqs) - 1;
1855        for (i = 0; i < cp->num_cqs; i++) {
1856                ictx->cstorm_st_context.cq_c_prod_sqn_arr.sqn[i] =
1857                        ISCSI_INITIAL_SN;
1858                ictx->cstorm_st_context.cq_c_sqn_2_notify_arr.sqn[i] =
1859                        ISCSI_INITIAL_SN;
1860        }
1861
1862        ictx->xstorm_ag_context.cdu_reserved =
1863                CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG,
1864                                       ISCSI_CONNECTION_TYPE);
1865        ictx->ustorm_ag_context.cdu_usage =
1866                CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG,
1867                                       ISCSI_CONNECTION_TYPE);
1868        return 0;
1869
1870}
1871
1872static int cnic_bnx2x_iscsi_ofld1(struct cnic_dev *dev, struct kwqe *wqes[],
1873                                   u32 num, int *work)
1874{
1875        struct iscsi_kwqe_conn_offload1 *req1;
1876        struct iscsi_kwqe_conn_offload2 *req2;
1877        struct cnic_local *cp = dev->cnic_priv;
1878        struct bnx2x *bp = netdev_priv(dev->netdev);
1879        struct cnic_context *ctx;
1880        struct iscsi_kcqe kcqe;
1881        struct kcqe *cqes[1];
1882        u32 l5_cid;
1883        int ret = 0;
1884
1885        if (num < 2) {
1886                *work = num;
1887                return -EINVAL;
1888        }
1889
1890        req1 = (struct iscsi_kwqe_conn_offload1 *) wqes[0];
1891        req2 = (struct iscsi_kwqe_conn_offload2 *) wqes[1];
1892        if ((num - 2) < req2->num_additional_wqes) {
1893                *work = num;
1894                return -EINVAL;
1895        }
1896        *work = 2 + req2->num_additional_wqes;
1897
1898        l5_cid = req1->iscsi_conn_id;
1899        if (l5_cid >= MAX_ISCSI_TBL_SZ)
1900                return -EINVAL;
1901
1902        memset(&kcqe, 0, sizeof(kcqe));
1903        kcqe.op_code = ISCSI_KCQE_OPCODE_OFFLOAD_CONN;
1904        kcqe.iscsi_conn_id = l5_cid;
1905        kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE;
1906
1907        ctx = &cp->ctx_tbl[l5_cid];
1908        if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags)) {
1909                kcqe.completion_status =
1910                        ISCSI_KCQE_COMPLETION_STATUS_CID_BUSY;
1911                goto done;
1912        }
1913
1914        if (atomic_inc_return(&cp->iscsi_conn) > dev->max_iscsi_conn) {
1915                atomic_dec(&cp->iscsi_conn);
1916                goto done;
1917        }
1918        ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid);
1919        if (ret) {
1920                atomic_dec(&cp->iscsi_conn);
1921                ret = 0;
1922                goto done;
1923        }
1924        ret = cnic_setup_bnx2x_ctx(dev, wqes, num);
1925        if (ret < 0) {
1926                cnic_free_bnx2x_conn_resc(dev, l5_cid);
1927                atomic_dec(&cp->iscsi_conn);
1928                goto done;
1929        }
1930
1931        kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS;
1932        kcqe.iscsi_conn_context_id = BNX2X_HW_CID(bp, cp->ctx_tbl[l5_cid].cid);
1933
1934done:
1935        cqes[0] = (struct kcqe *) &kcqe;
1936        cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1);
1937        return 0;
1938}
1939
1940
1941static int cnic_bnx2x_iscsi_update(struct cnic_dev *dev, struct kwqe *kwqe)
1942{
1943        struct cnic_local *cp = dev->cnic_priv;
1944        struct iscsi_kwqe_conn_update *req =
1945                (struct iscsi_kwqe_conn_update *) kwqe;
1946        void *data;
1947        union l5cm_specific_data l5_data;
1948        u32 l5_cid, cid = BNX2X_SW_CID(req->context_id);
1949        int ret;
1950
1951        if (cnic_get_l5_cid(cp, cid, &l5_cid) != 0)
1952                return -EINVAL;
1953
1954        data = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data);
1955        if (!data)
1956                return -ENOMEM;
1957
1958        memcpy(data, kwqe, sizeof(struct kwqe));
1959
1960        ret = cnic_submit_kwqe_16(dev, ISCSI_RAMROD_CMD_ID_UPDATE_CONN,
1961                        req->context_id, ISCSI_CONNECTION_TYPE, &l5_data);
1962        return ret;
1963}
1964
1965static int cnic_bnx2x_destroy_ramrod(struct cnic_dev *dev, u32 l5_cid)
1966{
1967        struct cnic_local *cp = dev->cnic_priv;
1968        struct bnx2x *bp = netdev_priv(dev->netdev);
1969        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
1970        union l5cm_specific_data l5_data;
1971        int ret;
1972        u32 hw_cid;
1973
1974        init_waitqueue_head(&ctx->waitq);
1975        ctx->wait_cond = 0;
1976        memset(&l5_data, 0, sizeof(l5_data));
1977        hw_cid = BNX2X_HW_CID(bp, ctx->cid);
1978
1979        ret = cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL,
1980                                  hw_cid, NONE_CONNECTION_TYPE, &l5_data);
1981
1982        if (ret == 0) {
1983                wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO);
1984                if (unlikely(test_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags)))
1985                        return -EBUSY;
1986        }
1987
1988        return 0;
1989}
1990
1991static int cnic_bnx2x_iscsi_destroy(struct cnic_dev *dev, struct kwqe *kwqe)
1992{
1993        struct cnic_local *cp = dev->cnic_priv;
1994        struct iscsi_kwqe_conn_destroy *req =
1995                (struct iscsi_kwqe_conn_destroy *) kwqe;
1996        u32 l5_cid = req->reserved0;
1997        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
1998        int ret = 0;
1999        struct iscsi_kcqe kcqe;
2000        struct kcqe *cqes[1];
2001
2002        if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags))
2003                goto skip_cfc_delete;
2004
2005        if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) {
2006                unsigned long delta = ctx->timestamp + (2 * HZ) - jiffies;
2007
2008                if (delta > (2 * HZ))
2009                        delta = 0;
2010
2011                set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags);
2012                queue_delayed_work(cnic_wq, &cp->delete_task, delta);
2013                goto destroy_reply;
2014        }
2015
2016        ret = cnic_bnx2x_destroy_ramrod(dev, l5_cid);
2017
2018skip_cfc_delete:
2019        cnic_free_bnx2x_conn_resc(dev, l5_cid);
2020
2021        if (!ret) {
2022                atomic_dec(&cp->iscsi_conn);
2023                clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags);
2024        }
2025
2026destroy_reply:
2027        memset(&kcqe, 0, sizeof(kcqe));
2028        kcqe.op_code = ISCSI_KCQE_OPCODE_DESTROY_CONN;
2029        kcqe.iscsi_conn_id = l5_cid;
2030        kcqe.completion_status = ISCSI_KCQE_COMPLETION_STATUS_SUCCESS;
2031        kcqe.iscsi_conn_context_id = req->context_id;
2032
2033        cqes[0] = (struct kcqe *) &kcqe;
2034        cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_ISCSI, cqes, 1);
2035
2036        return 0;
2037}
2038
2039static void cnic_init_storm_conn_bufs(struct cnic_dev *dev,
2040                                      struct l4_kwq_connect_req1 *kwqe1,
2041                                      struct l4_kwq_connect_req3 *kwqe3,
2042                                      struct l5cm_active_conn_buffer *conn_buf)
2043{
2044        struct l5cm_conn_addr_params *conn_addr = &conn_buf->conn_addr_buf;
2045        struct l5cm_xstorm_conn_buffer *xstorm_buf =
2046                &conn_buf->xstorm_conn_buffer;
2047        struct l5cm_tstorm_conn_buffer *tstorm_buf =
2048                &conn_buf->tstorm_conn_buffer;
2049        struct regpair context_addr;
2050        u32 cid = BNX2X_SW_CID(kwqe1->cid);
2051        struct in6_addr src_ip, dst_ip;
2052        int i;
2053        u32 *addrp;
2054
2055        addrp = (u32 *) &conn_addr->local_ip_addr;
2056        for (i = 0; i < 4; i++, addrp++)
2057                src_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp);
2058
2059        addrp = (u32 *) &conn_addr->remote_ip_addr;
2060        for (i = 0; i < 4; i++, addrp++)
2061                dst_ip.in6_u.u6_addr32[i] = cpu_to_be32(*addrp);
2062
2063        cnic_get_bnx2x_ctx(dev, cid, 0, &context_addr);
2064
2065        xstorm_buf->context_addr.hi = context_addr.hi;
2066        xstorm_buf->context_addr.lo = context_addr.lo;
2067        xstorm_buf->mss = 0xffff;
2068        xstorm_buf->rcv_buf = kwqe3->rcv_buf;
2069        if (kwqe1->tcp_flags & L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE)
2070                xstorm_buf->params |= L5CM_XSTORM_CONN_BUFFER_NAGLE_ENABLE;
2071        xstorm_buf->pseudo_header_checksum =
2072                swab16(~csum_ipv6_magic(&src_ip, &dst_ip, 0, IPPROTO_TCP, 0));
2073
2074        if (kwqe3->ka_timeout) {
2075                tstorm_buf->ka_enable = 1;
2076                tstorm_buf->ka_timeout = kwqe3->ka_timeout;
2077                tstorm_buf->ka_interval = kwqe3->ka_interval;
2078                tstorm_buf->ka_max_probe_count = kwqe3->ka_max_probe_count;
2079        }
2080        tstorm_buf->max_rt_time = 0xffffffff;
2081}
2082
2083static void cnic_init_bnx2x_mac(struct cnic_dev *dev)
2084{
2085        struct bnx2x *bp = netdev_priv(dev->netdev);
2086        u32 pfid = bp->pfid;
2087        u8 *mac = dev->mac_addr;
2088
2089        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
2090                 XSTORM_ISCSI_LOCAL_MAC_ADDR0_OFFSET(pfid), mac[0]);
2091        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
2092                 XSTORM_ISCSI_LOCAL_MAC_ADDR1_OFFSET(pfid), mac[1]);
2093        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
2094                 XSTORM_ISCSI_LOCAL_MAC_ADDR2_OFFSET(pfid), mac[2]);
2095        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
2096                 XSTORM_ISCSI_LOCAL_MAC_ADDR3_OFFSET(pfid), mac[3]);
2097        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
2098                 XSTORM_ISCSI_LOCAL_MAC_ADDR4_OFFSET(pfid), mac[4]);
2099        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
2100                 XSTORM_ISCSI_LOCAL_MAC_ADDR5_OFFSET(pfid), mac[5]);
2101
2102        CNIC_WR8(dev, BAR_TSTRORM_INTMEM +
2103                 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[5]);
2104        CNIC_WR8(dev, BAR_TSTRORM_INTMEM +
2105                 TSTORM_ISCSI_TCP_VARS_LSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1,
2106                 mac[4]);
2107        CNIC_WR8(dev, BAR_TSTRORM_INTMEM +
2108                 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid), mac[3]);
2109        CNIC_WR8(dev, BAR_TSTRORM_INTMEM +
2110                 TSTORM_ISCSI_TCP_VARS_MID_LOCAL_MAC_ADDR_OFFSET(pfid) + 1,
2111                 mac[2]);
2112        CNIC_WR8(dev, BAR_TSTRORM_INTMEM +
2113                 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid), mac[1]);
2114        CNIC_WR8(dev, BAR_TSTRORM_INTMEM +
2115                 TSTORM_ISCSI_TCP_VARS_MSB_LOCAL_MAC_ADDR_OFFSET(pfid) + 1,
2116                 mac[0]);
2117}
2118
2119static int cnic_bnx2x_connect(struct cnic_dev *dev, struct kwqe *wqes[],
2120                              u32 num, int *work)
2121{
2122        struct cnic_local *cp = dev->cnic_priv;
2123        struct bnx2x *bp = netdev_priv(dev->netdev);
2124        struct l4_kwq_connect_req1 *kwqe1 =
2125                (struct l4_kwq_connect_req1 *) wqes[0];
2126        struct l4_kwq_connect_req3 *kwqe3;
2127        struct l5cm_active_conn_buffer *conn_buf;
2128        struct l5cm_conn_addr_params *conn_addr;
2129        union l5cm_specific_data l5_data;
2130        u32 l5_cid = kwqe1->pg_cid;
2131        struct cnic_sock *csk = &cp->csk_tbl[l5_cid];
2132        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
2133        int ret;
2134
2135        if (num < 2) {
2136                *work = num;
2137                return -EINVAL;
2138        }
2139
2140        if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6)
2141                *work = 3;
2142        else
2143                *work = 2;
2144
2145        if (num < *work) {
2146                *work = num;
2147                return -EINVAL;
2148        }
2149
2150        if (sizeof(*conn_buf) > CNIC_KWQ16_DATA_SIZE) {
2151                netdev_err(dev->netdev, "conn_buf size too big\n");
2152                return -ENOMEM;
2153        }
2154        conn_buf = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data);
2155        if (!conn_buf)
2156                return -ENOMEM;
2157
2158        memset(conn_buf, 0, sizeof(*conn_buf));
2159
2160        conn_addr = &conn_buf->conn_addr_buf;
2161        conn_addr->remote_addr_0 = csk->ha[0];
2162        conn_addr->remote_addr_1 = csk->ha[1];
2163        conn_addr->remote_addr_2 = csk->ha[2];
2164        conn_addr->remote_addr_3 = csk->ha[3];
2165        conn_addr->remote_addr_4 = csk->ha[4];
2166        conn_addr->remote_addr_5 = csk->ha[5];
2167
2168        if (kwqe1->conn_flags & L4_KWQ_CONNECT_REQ1_IP_V6) {
2169                struct l4_kwq_connect_req2 *kwqe2 =
2170                        (struct l4_kwq_connect_req2 *) wqes[1];
2171
2172                conn_addr->local_ip_addr.ip_addr_hi_hi = kwqe2->src_ip_v6_4;
2173                conn_addr->local_ip_addr.ip_addr_hi_lo = kwqe2->src_ip_v6_3;
2174                conn_addr->local_ip_addr.ip_addr_lo_hi = kwqe2->src_ip_v6_2;
2175
2176                conn_addr->remote_ip_addr.ip_addr_hi_hi = kwqe2->dst_ip_v6_4;
2177                conn_addr->remote_ip_addr.ip_addr_hi_lo = kwqe2->dst_ip_v6_3;
2178                conn_addr->remote_ip_addr.ip_addr_lo_hi = kwqe2->dst_ip_v6_2;
2179                conn_addr->params |= L5CM_CONN_ADDR_PARAMS_IP_VERSION;
2180        }
2181        kwqe3 = (struct l4_kwq_connect_req3 *) wqes[*work - 1];
2182
2183        conn_addr->local_ip_addr.ip_addr_lo_lo = kwqe1->src_ip;
2184        conn_addr->remote_ip_addr.ip_addr_lo_lo = kwqe1->dst_ip;
2185        conn_addr->local_tcp_port = kwqe1->src_port;
2186        conn_addr->remote_tcp_port = kwqe1->dst_port;
2187
2188        conn_addr->pmtu = kwqe3->pmtu;
2189        cnic_init_storm_conn_bufs(dev, kwqe1, kwqe3, conn_buf);
2190
2191        CNIC_WR16(dev, BAR_XSTRORM_INTMEM +
2192                  XSTORM_ISCSI_LOCAL_VLAN_OFFSET(bp->pfid), csk->vlan_id);
2193
2194        ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_TCP_CONNECT,
2195                        kwqe1->cid, ISCSI_CONNECTION_TYPE, &l5_data);
2196        if (!ret)
2197                set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags);
2198
2199        return ret;
2200}
2201
2202static int cnic_bnx2x_close(struct cnic_dev *dev, struct kwqe *kwqe)
2203{
2204        struct l4_kwq_close_req *req = (struct l4_kwq_close_req *) kwqe;
2205        union l5cm_specific_data l5_data;
2206        int ret;
2207
2208        memset(&l5_data, 0, sizeof(l5_data));
2209        ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_CLOSE,
2210                        req->cid, ISCSI_CONNECTION_TYPE, &l5_data);
2211        return ret;
2212}
2213
2214static int cnic_bnx2x_reset(struct cnic_dev *dev, struct kwqe *kwqe)
2215{
2216        struct l4_kwq_reset_req *req = (struct l4_kwq_reset_req *) kwqe;
2217        union l5cm_specific_data l5_data;
2218        int ret;
2219
2220        memset(&l5_data, 0, sizeof(l5_data));
2221        ret = cnic_submit_kwqe_16(dev, L5CM_RAMROD_CMD_ID_ABORT,
2222                        req->cid, ISCSI_CONNECTION_TYPE, &l5_data);
2223        return ret;
2224}
2225static int cnic_bnx2x_offload_pg(struct cnic_dev *dev, struct kwqe *kwqe)
2226{
2227        struct l4_kwq_offload_pg *req = (struct l4_kwq_offload_pg *) kwqe;
2228        struct l4_kcq kcqe;
2229        struct kcqe *cqes[1];
2230
2231        memset(&kcqe, 0, sizeof(kcqe));
2232        kcqe.pg_host_opaque = req->host_opaque;
2233        kcqe.pg_cid = req->host_opaque;
2234        kcqe.op_code = L4_KCQE_OPCODE_VALUE_OFFLOAD_PG;
2235        cqes[0] = (struct kcqe *) &kcqe;
2236        cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1);
2237        return 0;
2238}
2239
2240static int cnic_bnx2x_update_pg(struct cnic_dev *dev, struct kwqe *kwqe)
2241{
2242        struct l4_kwq_update_pg *req = (struct l4_kwq_update_pg *) kwqe;
2243        struct l4_kcq kcqe;
2244        struct kcqe *cqes[1];
2245
2246        memset(&kcqe, 0, sizeof(kcqe));
2247        kcqe.pg_host_opaque = req->pg_host_opaque;
2248        kcqe.pg_cid = req->pg_cid;
2249        kcqe.op_code = L4_KCQE_OPCODE_VALUE_UPDATE_PG;
2250        cqes[0] = (struct kcqe *) &kcqe;
2251        cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_L4, cqes, 1);
2252        return 0;
2253}
2254
2255static int cnic_bnx2x_fcoe_stat(struct cnic_dev *dev, struct kwqe *kwqe)
2256{
2257        struct fcoe_kwqe_stat *req;
2258        struct fcoe_stat_ramrod_params *fcoe_stat;
2259        union l5cm_specific_data l5_data;
2260        struct cnic_local *cp = dev->cnic_priv;
2261        struct bnx2x *bp = netdev_priv(dev->netdev);
2262        int ret;
2263        u32 cid;
2264
2265        req = (struct fcoe_kwqe_stat *) kwqe;
2266        cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid);
2267
2268        fcoe_stat = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data);
2269        if (!fcoe_stat)
2270                return -ENOMEM;
2271
2272        memset(fcoe_stat, 0, sizeof(*fcoe_stat));
2273        memcpy(&fcoe_stat->stat_kwqe, req, sizeof(*req));
2274
2275        ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_STAT_FUNC, cid,
2276                                  FCOE_CONNECTION_TYPE, &l5_data);
2277        return ret;
2278}
2279
2280static int cnic_bnx2x_fcoe_init1(struct cnic_dev *dev, struct kwqe *wqes[],
2281                                 u32 num, int *work)
2282{
2283        int ret;
2284        struct cnic_local *cp = dev->cnic_priv;
2285        struct bnx2x *bp = netdev_priv(dev->netdev);
2286        u32 cid;
2287        struct fcoe_init_ramrod_params *fcoe_init;
2288        struct fcoe_kwqe_init1 *req1;
2289        struct fcoe_kwqe_init2 *req2;
2290        struct fcoe_kwqe_init3 *req3;
2291        union l5cm_specific_data l5_data;
2292
2293        if (num < 3) {
2294                *work = num;
2295                return -EINVAL;
2296        }
2297        req1 = (struct fcoe_kwqe_init1 *) wqes[0];
2298        req2 = (struct fcoe_kwqe_init2 *) wqes[1];
2299        req3 = (struct fcoe_kwqe_init3 *) wqes[2];
2300        if (req2->hdr.op_code != FCOE_KWQE_OPCODE_INIT2) {
2301                *work = 1;
2302                return -EINVAL;
2303        }
2304        if (req3->hdr.op_code != FCOE_KWQE_OPCODE_INIT3) {
2305                *work = 2;
2306                return -EINVAL;
2307        }
2308
2309        if (sizeof(*fcoe_init) > CNIC_KWQ16_DATA_SIZE) {
2310                netdev_err(dev->netdev, "fcoe_init size too big\n");
2311                return -ENOMEM;
2312        }
2313        fcoe_init = cnic_get_kwqe_16_data(cp, BNX2X_FCOE_L5_CID_BASE, &l5_data);
2314        if (!fcoe_init)
2315                return -ENOMEM;
2316
2317        memset(fcoe_init, 0, sizeof(*fcoe_init));
2318        memcpy(&fcoe_init->init_kwqe1, req1, sizeof(*req1));
2319        memcpy(&fcoe_init->init_kwqe2, req2, sizeof(*req2));
2320        memcpy(&fcoe_init->init_kwqe3, req3, sizeof(*req3));
2321        fcoe_init->eq_pbl_base.lo = cp->kcq2.dma.pgtbl_map & 0xffffffff;
2322        fcoe_init->eq_pbl_base.hi = (u64) cp->kcq2.dma.pgtbl_map >> 32;
2323        fcoe_init->eq_pbl_size = cp->kcq2.dma.num_pages;
2324
2325        fcoe_init->sb_num = cp->status_blk_num;
2326        fcoe_init->eq_prod = MAX_KCQ_IDX;
2327        fcoe_init->sb_id = HC_INDEX_FCOE_EQ_CONS;
2328        cp->kcq2.sw_prod_idx = 0;
2329
2330        cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid);
2331        ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_INIT_FUNC, cid,
2332                                  FCOE_CONNECTION_TYPE, &l5_data);
2333        *work = 3;
2334        return ret;
2335}
2336
2337static int cnic_bnx2x_fcoe_ofld1(struct cnic_dev *dev, struct kwqe *wqes[],
2338                                 u32 num, int *work)
2339{
2340        int ret = 0;
2341        u32 cid = -1, l5_cid;
2342        struct cnic_local *cp = dev->cnic_priv;
2343        struct bnx2x *bp = netdev_priv(dev->netdev);
2344        struct fcoe_kwqe_conn_offload1 *req1;
2345        struct fcoe_kwqe_conn_offload2 *req2;
2346        struct fcoe_kwqe_conn_offload3 *req3;
2347        struct fcoe_kwqe_conn_offload4 *req4;
2348        struct fcoe_conn_offload_ramrod_params *fcoe_offload;
2349        struct cnic_context *ctx;
2350        struct fcoe_context *fctx;
2351        struct regpair ctx_addr;
2352        union l5cm_specific_data l5_data;
2353        struct fcoe_kcqe kcqe;
2354        struct kcqe *cqes[1];
2355
2356        if (num < 4) {
2357                *work = num;
2358                return -EINVAL;
2359        }
2360        req1 = (struct fcoe_kwqe_conn_offload1 *) wqes[0];
2361        req2 = (struct fcoe_kwqe_conn_offload2 *) wqes[1];
2362        req3 = (struct fcoe_kwqe_conn_offload3 *) wqes[2];
2363        req4 = (struct fcoe_kwqe_conn_offload4 *) wqes[3];
2364
2365        *work = 4;
2366
2367        l5_cid = req1->fcoe_conn_id;
2368        if (l5_cid >= dev->max_fcoe_conn)
2369                goto err_reply;
2370
2371        l5_cid += BNX2X_FCOE_L5_CID_BASE;
2372
2373        ctx = &cp->ctx_tbl[l5_cid];
2374        if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags))
2375                goto err_reply;
2376
2377        ret = cnic_alloc_bnx2x_conn_resc(dev, l5_cid);
2378        if (ret) {
2379                ret = 0;
2380                goto err_reply;
2381        }
2382        cid = ctx->cid;
2383
2384        fctx = cnic_get_bnx2x_ctx(dev, cid, 1, &ctx_addr);
2385        if (fctx) {
2386                u32 hw_cid = BNX2X_HW_CID(bp, cid);
2387                u32 val;
2388
2389                val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_XCM_AG,
2390                                             FCOE_CONNECTION_TYPE);
2391                fctx->xstorm_ag_context.cdu_reserved = val;
2392                val = CDU_RSRVD_VALUE_TYPE_A(hw_cid, CDU_REGION_NUMBER_UCM_AG,
2393                                             FCOE_CONNECTION_TYPE);
2394                fctx->ustorm_ag_context.cdu_usage = val;
2395        }
2396        if (sizeof(*fcoe_offload) > CNIC_KWQ16_DATA_SIZE) {
2397                netdev_err(dev->netdev, "fcoe_offload size too big\n");
2398                goto err_reply;
2399        }
2400        fcoe_offload = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data);
2401        if (!fcoe_offload)
2402                goto err_reply;
2403
2404        memset(fcoe_offload, 0, sizeof(*fcoe_offload));
2405        memcpy(&fcoe_offload->offload_kwqe1, req1, sizeof(*req1));
2406        memcpy(&fcoe_offload->offload_kwqe2, req2, sizeof(*req2));
2407        memcpy(&fcoe_offload->offload_kwqe3, req3, sizeof(*req3));
2408        memcpy(&fcoe_offload->offload_kwqe4, req4, sizeof(*req4));
2409
2410        cid = BNX2X_HW_CID(bp, cid);
2411        ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_OFFLOAD_CONN, cid,
2412                                  FCOE_CONNECTION_TYPE, &l5_data);
2413        if (!ret)
2414                set_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags);
2415
2416        return ret;
2417
2418err_reply:
2419        if (cid != -1)
2420                cnic_free_bnx2x_conn_resc(dev, l5_cid);
2421
2422        memset(&kcqe, 0, sizeof(kcqe));
2423        kcqe.op_code = FCOE_KCQE_OPCODE_OFFLOAD_CONN;
2424        kcqe.fcoe_conn_id = req1->fcoe_conn_id;
2425        kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAILURE;
2426
2427        cqes[0] = (struct kcqe *) &kcqe;
2428        cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1);
2429        return ret;
2430}
2431
2432static int cnic_bnx2x_fcoe_enable(struct cnic_dev *dev, struct kwqe *kwqe)
2433{
2434        struct fcoe_kwqe_conn_enable_disable *req;
2435        struct fcoe_conn_enable_disable_ramrod_params *fcoe_enable;
2436        union l5cm_specific_data l5_data;
2437        int ret;
2438        u32 cid, l5_cid;
2439        struct cnic_local *cp = dev->cnic_priv;
2440
2441        req = (struct fcoe_kwqe_conn_enable_disable *) kwqe;
2442        cid = req->context_id;
2443        l5_cid = req->conn_id + BNX2X_FCOE_L5_CID_BASE;
2444
2445        if (sizeof(*fcoe_enable) > CNIC_KWQ16_DATA_SIZE) {
2446                netdev_err(dev->netdev, "fcoe_enable size too big\n");
2447                return -ENOMEM;
2448        }
2449        fcoe_enable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data);
2450        if (!fcoe_enable)
2451                return -ENOMEM;
2452
2453        memset(fcoe_enable, 0, sizeof(*fcoe_enable));
2454        memcpy(&fcoe_enable->enable_disable_kwqe, req, sizeof(*req));
2455        ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_ENABLE_CONN, cid,
2456                                  FCOE_CONNECTION_TYPE, &l5_data);
2457        return ret;
2458}
2459
2460static int cnic_bnx2x_fcoe_disable(struct cnic_dev *dev, struct kwqe *kwqe)
2461{
2462        struct fcoe_kwqe_conn_enable_disable *req;
2463        struct fcoe_conn_enable_disable_ramrod_params *fcoe_disable;
2464        union l5cm_specific_data l5_data;
2465        int ret;
2466        u32 cid, l5_cid;
2467        struct cnic_local *cp = dev->cnic_priv;
2468
2469        req = (struct fcoe_kwqe_conn_enable_disable *) kwqe;
2470        cid = req->context_id;
2471        l5_cid = req->conn_id;
2472        if (l5_cid >= dev->max_fcoe_conn)
2473                return -EINVAL;
2474
2475        l5_cid += BNX2X_FCOE_L5_CID_BASE;
2476
2477        if (sizeof(*fcoe_disable) > CNIC_KWQ16_DATA_SIZE) {
2478                netdev_err(dev->netdev, "fcoe_disable size too big\n");
2479                return -ENOMEM;
2480        }
2481        fcoe_disable = cnic_get_kwqe_16_data(cp, l5_cid, &l5_data);
2482        if (!fcoe_disable)
2483                return -ENOMEM;
2484
2485        memset(fcoe_disable, 0, sizeof(*fcoe_disable));
2486        memcpy(&fcoe_disable->enable_disable_kwqe, req, sizeof(*req));
2487        ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DISABLE_CONN, cid,
2488                                  FCOE_CONNECTION_TYPE, &l5_data);
2489        return ret;
2490}
2491
2492static int cnic_bnx2x_fcoe_destroy(struct cnic_dev *dev, struct kwqe *kwqe)
2493{
2494        struct fcoe_kwqe_conn_destroy *req;
2495        union l5cm_specific_data l5_data;
2496        int ret;
2497        u32 cid, l5_cid;
2498        struct cnic_local *cp = dev->cnic_priv;
2499        struct cnic_context *ctx;
2500        struct fcoe_kcqe kcqe;
2501        struct kcqe *cqes[1];
2502
2503        req = (struct fcoe_kwqe_conn_destroy *) kwqe;
2504        cid = req->context_id;
2505        l5_cid = req->conn_id;
2506        if (l5_cid >= dev->max_fcoe_conn)
2507                return -EINVAL;
2508
2509        l5_cid += BNX2X_FCOE_L5_CID_BASE;
2510
2511        ctx = &cp->ctx_tbl[l5_cid];
2512
2513        init_waitqueue_head(&ctx->waitq);
2514        ctx->wait_cond = 0;
2515
2516        memset(&kcqe, 0, sizeof(kcqe));
2517        kcqe.completion_status = FCOE_KCQE_COMPLETION_STATUS_ERROR;
2518        memset(&l5_data, 0, sizeof(l5_data));
2519        ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_TERMINATE_CONN, cid,
2520                                  FCOE_CONNECTION_TYPE, &l5_data);
2521        if (ret == 0) {
2522                wait_event_timeout(ctx->waitq, ctx->wait_cond, CNIC_RAMROD_TMO);
2523                if (ctx->wait_cond)
2524                        kcqe.completion_status = 0;
2525        }
2526
2527        set_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags);
2528        queue_delayed_work(cnic_wq, &cp->delete_task, msecs_to_jiffies(2000));
2529
2530        kcqe.op_code = FCOE_KCQE_OPCODE_DESTROY_CONN;
2531        kcqe.fcoe_conn_id = req->conn_id;
2532        kcqe.fcoe_conn_context_id = cid;
2533
2534        cqes[0] = (struct kcqe *) &kcqe;
2535        cnic_reply_bnx2x_kcqes(dev, CNIC_ULP_FCOE, cqes, 1);
2536        return ret;
2537}
2538
2539static void cnic_bnx2x_delete_wait(struct cnic_dev *dev, u32 start_cid)
2540{
2541        struct cnic_local *cp = dev->cnic_priv;
2542        u32 i;
2543
2544        for (i = start_cid; i < cp->max_cid_space; i++) {
2545                struct cnic_context *ctx = &cp->ctx_tbl[i];
2546                int j;
2547
2548                while (test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags))
2549                        msleep(10);
2550
2551                for (j = 0; j < 5; j++) {
2552                        if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags))
2553                                break;
2554                        msleep(20);
2555                }
2556
2557                if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags))
2558                        netdev_warn(dev->netdev, "CID %x not deleted\n",
2559                                   ctx->cid);
2560        }
2561}
2562
2563static int cnic_bnx2x_fcoe_fw_destroy(struct cnic_dev *dev, struct kwqe *kwqe)
2564{
2565        struct fcoe_kwqe_destroy *req;
2566        union l5cm_specific_data l5_data;
2567        struct cnic_local *cp = dev->cnic_priv;
2568        struct bnx2x *bp = netdev_priv(dev->netdev);
2569        int ret;
2570        u32 cid;
2571
2572        cnic_bnx2x_delete_wait(dev, MAX_ISCSI_TBL_SZ);
2573
2574        req = (struct fcoe_kwqe_destroy *) kwqe;
2575        cid = BNX2X_HW_CID(bp, cp->fcoe_init_cid);
2576
2577        memset(&l5_data, 0, sizeof(l5_data));
2578        ret = cnic_submit_kwqe_16(dev, FCOE_RAMROD_CMD_ID_DESTROY_FUNC, cid,
2579                                  FCOE_CONNECTION_TYPE, &l5_data);
2580        return ret;
2581}
2582
2583static void cnic_bnx2x_kwqe_err(struct cnic_dev *dev, struct kwqe *kwqe)
2584{
2585        struct cnic_local *cp = dev->cnic_priv;
2586        struct kcqe kcqe;
2587        struct kcqe *cqes[1];
2588        u32 cid;
2589        u32 opcode = KWQE_OPCODE(kwqe->kwqe_op_flag);
2590        u32 layer_code = kwqe->kwqe_op_flag & KWQE_LAYER_MASK;
2591        u32 kcqe_op;
2592        int ulp_type;
2593
2594        cid = kwqe->kwqe_info0;
2595        memset(&kcqe, 0, sizeof(kcqe));
2596
2597        if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_FCOE) {
2598                u32 l5_cid = 0;
2599
2600                ulp_type = CNIC_ULP_FCOE;
2601                if (opcode == FCOE_KWQE_OPCODE_DISABLE_CONN) {
2602                        struct fcoe_kwqe_conn_enable_disable *req;
2603
2604                        req = (struct fcoe_kwqe_conn_enable_disable *) kwqe;
2605                        kcqe_op = FCOE_KCQE_OPCODE_DISABLE_CONN;
2606                        cid = req->context_id;
2607                        l5_cid = req->conn_id;
2608                } else if (opcode == FCOE_KWQE_OPCODE_DESTROY) {
2609                        kcqe_op = FCOE_KCQE_OPCODE_DESTROY_FUNC;
2610                } else {
2611                        return;
2612                }
2613                kcqe.kcqe_op_flag = kcqe_op << KCQE_FLAGS_OPCODE_SHIFT;
2614                kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_FCOE;
2615                kcqe.kcqe_info1 = FCOE_KCQE_COMPLETION_STATUS_PARITY_ERROR;
2616                kcqe.kcqe_info2 = cid;
2617                kcqe.kcqe_info0 = l5_cid;
2618
2619        } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L5_ISCSI) {
2620                ulp_type = CNIC_ULP_ISCSI;
2621                if (opcode == ISCSI_KWQE_OPCODE_UPDATE_CONN)
2622                        cid = kwqe->kwqe_info1;
2623
2624                kcqe.kcqe_op_flag = (opcode + 0x10) << KCQE_FLAGS_OPCODE_SHIFT;
2625                kcqe.kcqe_op_flag |= KCQE_FLAGS_LAYER_MASK_L5_ISCSI;
2626                kcqe.kcqe_info1 = ISCSI_KCQE_COMPLETION_STATUS_PARITY_ERR;
2627                kcqe.kcqe_info2 = cid;
2628                cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &kcqe.kcqe_info0);
2629
2630        } else if (layer_code == KWQE_FLAGS_LAYER_MASK_L4) {
2631                struct l4_kcq *l4kcqe = (struct l4_kcq *) &kcqe;
2632
2633                ulp_type = CNIC_ULP_L4;
2634                if (opcode == L4_KWQE_OPCODE_VALUE_CONNECT1)
2635                        kcqe_op = L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE;
2636                else if (opcode == L4_KWQE_OPCODE_VALUE_RESET)
2637                        kcqe_op = L4_KCQE_OPCODE_VALUE_RESET_COMP;
2638                else if (opcode == L4_KWQE_OPCODE_VALUE_CLOSE)
2639                        kcqe_op = L4_KCQE_OPCODE_VALUE_CLOSE_COMP;
2640                else
2641                        return;
2642
2643                kcqe.kcqe_op_flag = (kcqe_op << KCQE_FLAGS_OPCODE_SHIFT) |
2644                                    KCQE_FLAGS_LAYER_MASK_L4;
2645                l4kcqe->status = L4_KCQE_COMPLETION_STATUS_PARITY_ERROR;
2646                l4kcqe->cid = cid;
2647                cnic_get_l5_cid(cp, BNX2X_SW_CID(cid), &l4kcqe->conn_id);
2648        } else {
2649                return;
2650        }
2651
2652        cqes[0] = &kcqe;
2653        cnic_reply_bnx2x_kcqes(dev, ulp_type, cqes, 1);
2654}
2655
2656static int cnic_submit_bnx2x_iscsi_kwqes(struct cnic_dev *dev,
2657                                         struct kwqe *wqes[], u32 num_wqes)
2658{
2659        int i, work, ret;
2660        u32 opcode;
2661        struct kwqe *kwqe;
2662
2663        if (!test_bit(CNIC_F_CNIC_UP, &dev->flags))
2664                return -EAGAIN;         /* bnx2 is down */
2665
2666        for (i = 0; i < num_wqes; ) {
2667                kwqe = wqes[i];
2668                opcode = KWQE_OPCODE(kwqe->kwqe_op_flag);
2669                work = 1;
2670
2671                switch (opcode) {
2672                case ISCSI_KWQE_OPCODE_INIT1:
2673                        ret = cnic_bnx2x_iscsi_init1(dev, kwqe);
2674                        break;
2675                case ISCSI_KWQE_OPCODE_INIT2:
2676                        ret = cnic_bnx2x_iscsi_init2(dev, kwqe);
2677                        break;
2678                case ISCSI_KWQE_OPCODE_OFFLOAD_CONN1:
2679                        ret = cnic_bnx2x_iscsi_ofld1(dev, &wqes[i],
2680                                                     num_wqes - i, &work);
2681                        break;
2682                case ISCSI_KWQE_OPCODE_UPDATE_CONN:
2683                        ret = cnic_bnx2x_iscsi_update(dev, kwqe);
2684                        break;
2685                case ISCSI_KWQE_OPCODE_DESTROY_CONN:
2686                        ret = cnic_bnx2x_iscsi_destroy(dev, kwqe);
2687                        break;
2688                case L4_KWQE_OPCODE_VALUE_CONNECT1:
2689                        ret = cnic_bnx2x_connect(dev, &wqes[i], num_wqes - i,
2690                                                 &work);
2691                        break;
2692                case L4_KWQE_OPCODE_VALUE_CLOSE:
2693                        ret = cnic_bnx2x_close(dev, kwqe);
2694                        break;
2695                case L4_KWQE_OPCODE_VALUE_RESET:
2696                        ret = cnic_bnx2x_reset(dev, kwqe);
2697                        break;
2698                case L4_KWQE_OPCODE_VALUE_OFFLOAD_PG:
2699                        ret = cnic_bnx2x_offload_pg(dev, kwqe);
2700                        break;
2701                case L4_KWQE_OPCODE_VALUE_UPDATE_PG:
2702                        ret = cnic_bnx2x_update_pg(dev, kwqe);
2703                        break;
2704                case L4_KWQE_OPCODE_VALUE_UPLOAD_PG:
2705                        ret = 0;
2706                        break;
2707                default:
2708                        ret = 0;
2709                        netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n",
2710                                   opcode);
2711                        break;
2712                }
2713                if (ret < 0) {
2714                        netdev_err(dev->netdev, "KWQE(0x%x) failed\n",
2715                                   opcode);
2716
2717                        /* Possibly bnx2x parity error, send completion
2718                         * to ulp drivers with error code to speed up
2719                         * cleanup and reset recovery.
2720                         */
2721                        if (ret == -EIO || ret == -EAGAIN)
2722                                cnic_bnx2x_kwqe_err(dev, kwqe);
2723                }
2724                i += work;
2725        }
2726        return 0;
2727}
2728
2729static int cnic_submit_bnx2x_fcoe_kwqes(struct cnic_dev *dev,
2730                                        struct kwqe *wqes[], u32 num_wqes)
2731{
2732        struct bnx2x *bp = netdev_priv(dev->netdev);
2733        int i, work, ret;
2734        u32 opcode;
2735        struct kwqe *kwqe;
2736
2737        if (!test_bit(CNIC_F_CNIC_UP, &dev->flags))
2738                return -EAGAIN;         /* bnx2 is down */
2739
2740        if (!BNX2X_CHIP_IS_E2_PLUS(bp))
2741                return -EINVAL;
2742
2743        for (i = 0; i < num_wqes; ) {
2744                kwqe = wqes[i];
2745                opcode = KWQE_OPCODE(kwqe->kwqe_op_flag);
2746                work = 1;
2747
2748                switch (opcode) {
2749                case FCOE_KWQE_OPCODE_INIT1:
2750                        ret = cnic_bnx2x_fcoe_init1(dev, &wqes[i],
2751                                                    num_wqes - i, &work);
2752                        break;
2753                case FCOE_KWQE_OPCODE_OFFLOAD_CONN1:
2754                        ret = cnic_bnx2x_fcoe_ofld1(dev, &wqes[i],
2755                                                    num_wqes - i, &work);
2756                        break;
2757                case FCOE_KWQE_OPCODE_ENABLE_CONN:
2758                        ret = cnic_bnx2x_fcoe_enable(dev, kwqe);
2759                        break;
2760                case FCOE_KWQE_OPCODE_DISABLE_CONN:
2761                        ret = cnic_bnx2x_fcoe_disable(dev, kwqe);
2762                        break;
2763                case FCOE_KWQE_OPCODE_DESTROY_CONN:
2764                        ret = cnic_bnx2x_fcoe_destroy(dev, kwqe);
2765                        break;
2766                case FCOE_KWQE_OPCODE_DESTROY:
2767                        ret = cnic_bnx2x_fcoe_fw_destroy(dev, kwqe);
2768                        break;
2769                case FCOE_KWQE_OPCODE_STAT:
2770                        ret = cnic_bnx2x_fcoe_stat(dev, kwqe);
2771                        break;
2772                default:
2773                        ret = 0;
2774                        netdev_err(dev->netdev, "Unknown type of KWQE(0x%x)\n",
2775                                   opcode);
2776                        break;
2777                }
2778                if (ret < 0) {
2779                        netdev_err(dev->netdev, "KWQE(0x%x) failed\n",
2780                                   opcode);
2781
2782                        /* Possibly bnx2x parity error, send completion
2783                         * to ulp drivers with error code to speed up
2784                         * cleanup and reset recovery.
2785                         */
2786                        if (ret == -EIO || ret == -EAGAIN)
2787                                cnic_bnx2x_kwqe_err(dev, kwqe);
2788                }
2789                i += work;
2790        }
2791        return 0;
2792}
2793
2794static int cnic_submit_bnx2x_kwqes(struct cnic_dev *dev, struct kwqe *wqes[],
2795                                   u32 num_wqes)
2796{
2797        int ret = -EINVAL;
2798        u32 layer_code;
2799
2800        if (!test_bit(CNIC_F_CNIC_UP, &dev->flags))
2801                return -EAGAIN;         /* bnx2x is down */
2802
2803        if (!num_wqes)
2804                return 0;
2805
2806        layer_code = wqes[0]->kwqe_op_flag & KWQE_LAYER_MASK;
2807        switch (layer_code) {
2808        case KWQE_FLAGS_LAYER_MASK_L5_ISCSI:
2809        case KWQE_FLAGS_LAYER_MASK_L4:
2810        case KWQE_FLAGS_LAYER_MASK_L2:
2811                ret = cnic_submit_bnx2x_iscsi_kwqes(dev, wqes, num_wqes);
2812                break;
2813
2814        case KWQE_FLAGS_LAYER_MASK_L5_FCOE:
2815                ret = cnic_submit_bnx2x_fcoe_kwqes(dev, wqes, num_wqes);
2816                break;
2817        }
2818        return ret;
2819}
2820
2821static inline u32 cnic_get_kcqe_layer_mask(u32 opflag)
2822{
2823        if (unlikely(KCQE_OPCODE(opflag) == FCOE_RAMROD_CMD_ID_TERMINATE_CONN))
2824                return KCQE_FLAGS_LAYER_MASK_L4;
2825
2826        return opflag & KCQE_FLAGS_LAYER_MASK;
2827}
2828
2829static void service_kcqes(struct cnic_dev *dev, int num_cqes)
2830{
2831        struct cnic_local *cp = dev->cnic_priv;
2832        int i, j, comp = 0;
2833
2834        i = 0;
2835        j = 1;
2836        while (num_cqes) {
2837                struct cnic_ulp_ops *ulp_ops;
2838                int ulp_type;
2839                u32 kcqe_op_flag = cp->completed_kcq[i]->kcqe_op_flag;
2840                u32 kcqe_layer = cnic_get_kcqe_layer_mask(kcqe_op_flag);
2841
2842                if (unlikely(kcqe_op_flag & KCQE_RAMROD_COMPLETION))
2843                        comp++;
2844
2845                while (j < num_cqes) {
2846                        u32 next_op = cp->completed_kcq[i + j]->kcqe_op_flag;
2847
2848                        if (cnic_get_kcqe_layer_mask(next_op) != kcqe_layer)
2849                                break;
2850
2851                        if (unlikely(next_op & KCQE_RAMROD_COMPLETION))
2852                                comp++;
2853                        j++;
2854                }
2855
2856                if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_RDMA)
2857                        ulp_type = CNIC_ULP_RDMA;
2858                else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_ISCSI)
2859                        ulp_type = CNIC_ULP_ISCSI;
2860                else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L5_FCOE)
2861                        ulp_type = CNIC_ULP_FCOE;
2862                else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L4)
2863                        ulp_type = CNIC_ULP_L4;
2864                else if (kcqe_layer == KCQE_FLAGS_LAYER_MASK_L2)
2865                        goto end;
2866                else {
2867                        netdev_err(dev->netdev, "Unknown type of KCQE(0x%x)\n",
2868                                   kcqe_op_flag);
2869                        goto end;
2870                }
2871
2872                rcu_read_lock();
2873                ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]);
2874                if (likely(ulp_ops)) {
2875                        ulp_ops->indicate_kcqes(cp->ulp_handle[ulp_type],
2876                                                  cp->completed_kcq + i, j);
2877                }
2878                rcu_read_unlock();
2879end:
2880                num_cqes -= j;
2881                i += j;
2882                j = 1;
2883        }
2884        if (unlikely(comp))
2885                cnic_spq_completion(dev, DRV_CTL_RET_L5_SPQ_CREDIT_CMD, comp);
2886}
2887
2888static int cnic_get_kcqes(struct cnic_dev *dev, struct kcq_info *info)
2889{
2890        struct cnic_local *cp = dev->cnic_priv;
2891        u16 i, ri, hw_prod, last;
2892        struct kcqe *kcqe;
2893        int kcqe_cnt = 0, last_cnt = 0;
2894
2895        i = ri = last = info->sw_prod_idx;
2896        ri &= MAX_KCQ_IDX;
2897        hw_prod = *info->hw_prod_idx_ptr;
2898        hw_prod = info->hw_idx(hw_prod);
2899
2900        while ((i != hw_prod) && (kcqe_cnt < MAX_COMPLETED_KCQE)) {
2901                kcqe = &info->kcq[KCQ_PG(ri)][KCQ_IDX(ri)];
2902                cp->completed_kcq[kcqe_cnt++] = kcqe;
2903                i = info->next_idx(i);
2904                ri = i & MAX_KCQ_IDX;
2905                if (likely(!(kcqe->kcqe_op_flag & KCQE_FLAGS_NEXT))) {
2906                        last_cnt = kcqe_cnt;
2907                        last = i;
2908                }
2909        }
2910
2911        info->sw_prod_idx = last;
2912        return last_cnt;
2913}
2914
2915static int cnic_l2_completion(struct cnic_local *cp)
2916{
2917        u16 hw_cons, sw_cons;
2918        struct cnic_uio_dev *udev = cp->udev;
2919        union eth_rx_cqe *cqe, *cqe_ring = (union eth_rx_cqe *)
2920                                        (udev->l2_ring + (2 * CNIC_PAGE_SIZE));
2921        u32 cmd;
2922        int comp = 0;
2923
2924        if (!test_bit(CNIC_F_BNX2X_CLASS, &cp->dev->flags))
2925                return 0;
2926
2927        hw_cons = *cp->rx_cons_ptr;
2928        if ((hw_cons & BNX2X_MAX_RCQ_DESC_CNT) == BNX2X_MAX_RCQ_DESC_CNT)
2929                hw_cons++;
2930
2931        sw_cons = cp->rx_cons;
2932        while (sw_cons != hw_cons) {
2933                u8 cqe_fp_flags;
2934
2935                cqe = &cqe_ring[sw_cons & BNX2X_MAX_RCQ_DESC_CNT];
2936                cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
2937                if (cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE) {
2938                        cmd = le32_to_cpu(cqe->ramrod_cqe.conn_and_cmd_data);
2939                        cmd >>= COMMON_RAMROD_ETH_RX_CQE_CMD_ID_SHIFT;
2940                        if (cmd == RAMROD_CMD_ID_ETH_CLIENT_SETUP ||
2941                            cmd == RAMROD_CMD_ID_ETH_HALT)
2942                                comp++;
2943                }
2944                sw_cons = BNX2X_NEXT_RCQE(sw_cons);
2945        }
2946        return comp;
2947}
2948
2949static void cnic_chk_pkt_rings(struct cnic_local *cp)
2950{
2951        u16 rx_cons, tx_cons;
2952        int comp = 0;
2953
2954        if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags))
2955                return;
2956
2957        rx_cons = *cp->rx_cons_ptr;
2958        tx_cons = *cp->tx_cons_ptr;
2959        if (cp->tx_cons != tx_cons || cp->rx_cons != rx_cons) {
2960                if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags))
2961                        comp = cnic_l2_completion(cp);
2962
2963                cp->tx_cons = tx_cons;
2964                cp->rx_cons = rx_cons;
2965
2966                if (cp->udev)
2967                        uio_event_notify(&cp->udev->cnic_uinfo);
2968        }
2969        if (comp)
2970                clear_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags);
2971}
2972
2973static u32 cnic_service_bnx2_queues(struct cnic_dev *dev)
2974{
2975        struct cnic_local *cp = dev->cnic_priv;
2976        u32 status_idx = (u16) *cp->kcq1.status_idx_ptr;
2977        int kcqe_cnt;
2978
2979        /* status block index must be read before reading other fields */
2980        rmb();
2981        cp->kwq_con_idx = *cp->kwq_con_idx_ptr;
2982
2983        while ((kcqe_cnt = cnic_get_kcqes(dev, &cp->kcq1))) {
2984
2985                service_kcqes(dev, kcqe_cnt);
2986
2987                /* Tell compiler that status_blk fields can change. */
2988                barrier();
2989                status_idx = (u16) *cp->kcq1.status_idx_ptr;
2990                /* status block index must be read first */
2991                rmb();
2992                cp->kwq_con_idx = *cp->kwq_con_idx_ptr;
2993        }
2994
2995        CNIC_WR16(dev, cp->kcq1.io_addr, cp->kcq1.sw_prod_idx);
2996
2997        cnic_chk_pkt_rings(cp);
2998
2999        return status_idx;
3000}
3001
3002static int cnic_service_bnx2(void *data, void *status_blk)
3003{
3004        struct cnic_dev *dev = data;
3005
3006        if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags))) {
3007                struct status_block *sblk = status_blk;
3008
3009                return sblk->status_idx;
3010        }
3011
3012        return cnic_service_bnx2_queues(dev);
3013}
3014
3015static void cnic_service_bnx2_msix(unsigned long data)
3016{
3017        struct cnic_dev *dev = (struct cnic_dev *) data;
3018        struct cnic_local *cp = dev->cnic_priv;
3019
3020        cp->last_status_idx = cnic_service_bnx2_queues(dev);
3021
3022        CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num |
3023                BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx);
3024}
3025
3026static void cnic_doirq(struct cnic_dev *dev)
3027{
3028        struct cnic_local *cp = dev->cnic_priv;
3029
3030        if (likely(test_bit(CNIC_F_CNIC_UP, &dev->flags))) {
3031                u16 prod = cp->kcq1.sw_prod_idx & MAX_KCQ_IDX;
3032
3033                prefetch(cp->status_blk.gen);
3034                prefetch(&cp->kcq1.kcq[KCQ_PG(prod)][KCQ_IDX(prod)]);
3035
3036                tasklet_schedule(&cp->cnic_irq_task);
3037        }
3038}
3039
3040static irqreturn_t cnic_irq(int irq, void *dev_instance)
3041{
3042        struct cnic_dev *dev = dev_instance;
3043        struct cnic_local *cp = dev->cnic_priv;
3044
3045        if (cp->ack_int)
3046                cp->ack_int(dev);
3047
3048        cnic_doirq(dev);
3049
3050        return IRQ_HANDLED;
3051}
3052
3053static inline void cnic_ack_bnx2x_int(struct cnic_dev *dev, u8 id, u8 storm,
3054                                      u16 index, u8 op, u8 update)
3055{
3056        struct bnx2x *bp = netdev_priv(dev->netdev);
3057        u32 hc_addr = (HC_REG_COMMAND_REG + BP_PORT(bp) * 32 +
3058                       COMMAND_REG_INT_ACK);
3059        struct igu_ack_register igu_ack;
3060
3061        igu_ack.status_block_index = index;
3062        igu_ack.sb_id_and_flags =
3063                        ((id << IGU_ACK_REGISTER_STATUS_BLOCK_ID_SHIFT) |
3064                         (storm << IGU_ACK_REGISTER_STORM_ID_SHIFT) |
3065                         (update << IGU_ACK_REGISTER_UPDATE_INDEX_SHIFT) |
3066                         (op << IGU_ACK_REGISTER_INTERRUPT_MODE_SHIFT));
3067
3068        CNIC_WR(dev, hc_addr, (*(u32 *)&igu_ack));
3069}
3070
3071static void cnic_ack_igu_sb(struct cnic_dev *dev, u8 igu_sb_id, u8 segment,
3072                            u16 index, u8 op, u8 update)
3073{
3074        struct igu_regular cmd_data;
3075        u32 igu_addr = BAR_IGU_INTMEM + (IGU_CMD_INT_ACK_BASE + igu_sb_id) * 8;
3076
3077        cmd_data.sb_id_and_flags =
3078                (index << IGU_REGULAR_SB_INDEX_SHIFT) |
3079                (segment << IGU_REGULAR_SEGMENT_ACCESS_SHIFT) |
3080                (update << IGU_REGULAR_BUPDATE_SHIFT) |
3081                (op << IGU_REGULAR_ENABLE_INT_SHIFT);
3082
3083
3084        CNIC_WR(dev, igu_addr, cmd_data.sb_id_and_flags);
3085}
3086
3087static void cnic_ack_bnx2x_msix(struct cnic_dev *dev)
3088{
3089        struct cnic_local *cp = dev->cnic_priv;
3090
3091        cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, 0,
3092                           IGU_INT_DISABLE, 0);
3093}
3094
3095static void cnic_ack_bnx2x_e2_msix(struct cnic_dev *dev)
3096{
3097        struct cnic_local *cp = dev->cnic_priv;
3098
3099        cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, 0,
3100                        IGU_INT_DISABLE, 0);
3101}
3102
3103static void cnic_arm_bnx2x_msix(struct cnic_dev *dev, u32 idx)
3104{
3105        struct cnic_local *cp = dev->cnic_priv;
3106
3107        cnic_ack_bnx2x_int(dev, cp->bnx2x_igu_sb_id, CSTORM_ID, idx,
3108                           IGU_INT_ENABLE, 1);
3109}
3110
3111static void cnic_arm_bnx2x_e2_msix(struct cnic_dev *dev, u32 idx)
3112{
3113        struct cnic_local *cp = dev->cnic_priv;
3114
3115        cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF, idx,
3116                        IGU_INT_ENABLE, 1);
3117}
3118
3119static u32 cnic_service_bnx2x_kcq(struct cnic_dev *dev, struct kcq_info *info)
3120{
3121        u32 last_status = *info->status_idx_ptr;
3122        int kcqe_cnt;
3123
3124        /* status block index must be read before reading the KCQ */
3125        rmb();
3126        while ((kcqe_cnt = cnic_get_kcqes(dev, info))) {
3127
3128                service_kcqes(dev, kcqe_cnt);
3129
3130                /* Tell compiler that sblk fields can change. */
3131                barrier();
3132
3133                last_status = *info->status_idx_ptr;
3134                /* status block index must be read before reading the KCQ */
3135                rmb();
3136        }
3137        return last_status;
3138}
3139
3140static void cnic_service_bnx2x_bh(unsigned long data)
3141{
3142        struct cnic_dev *dev = (struct cnic_dev *) data;
3143        struct cnic_local *cp = dev->cnic_priv;
3144        struct bnx2x *bp = netdev_priv(dev->netdev);
3145        u32 status_idx, new_status_idx;
3146
3147        if (unlikely(!test_bit(CNIC_F_CNIC_UP, &dev->flags)))
3148                return;
3149
3150        while (1) {
3151                status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq1);
3152
3153                CNIC_WR16(dev, cp->kcq1.io_addr,
3154                          cp->kcq1.sw_prod_idx + MAX_KCQ_IDX);
3155
3156                if (!CNIC_SUPPORTS_FCOE(bp)) {
3157                        cp->arm_int(dev, status_idx);
3158                        break;
3159                }
3160
3161                new_status_idx = cnic_service_bnx2x_kcq(dev, &cp->kcq2);
3162
3163                if (new_status_idx != status_idx)
3164                        continue;
3165
3166                CNIC_WR16(dev, cp->kcq2.io_addr, cp->kcq2.sw_prod_idx +
3167                          MAX_KCQ_IDX);
3168
3169                cnic_ack_igu_sb(dev, cp->bnx2x_igu_sb_id, IGU_SEG_ACCESS_DEF,
3170                                status_idx, IGU_INT_ENABLE, 1);
3171
3172                break;
3173        }
3174}
3175
3176static int cnic_service_bnx2x(void *data, void *status_blk)
3177{
3178        struct cnic_dev *dev = data;
3179        struct cnic_local *cp = dev->cnic_priv;
3180
3181        if (!(cp->ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX))
3182                cnic_doirq(dev);
3183
3184        cnic_chk_pkt_rings(cp);
3185
3186        return 0;
3187}
3188
3189static void cnic_ulp_stop_one(struct cnic_local *cp, int if_type)
3190{
3191        struct cnic_ulp_ops *ulp_ops;
3192
3193        if (if_type == CNIC_ULP_ISCSI)
3194                cnic_send_nlmsg(cp, ISCSI_KEVENT_IF_DOWN, NULL);
3195
3196        mutex_lock(&cnic_lock);
3197        ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type],
3198                                            lockdep_is_held(&cnic_lock));
3199        if (!ulp_ops) {
3200                mutex_unlock(&cnic_lock);
3201                return;
3202        }
3203        set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
3204        mutex_unlock(&cnic_lock);
3205
3206        if (test_and_clear_bit(ULP_F_START, &cp->ulp_flags[if_type]))
3207                ulp_ops->cnic_stop(cp->ulp_handle[if_type]);
3208
3209        clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
3210}
3211
3212static void cnic_ulp_stop(struct cnic_dev *dev)
3213{
3214        struct cnic_local *cp = dev->cnic_priv;
3215        int if_type;
3216
3217        for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++)
3218                cnic_ulp_stop_one(cp, if_type);
3219}
3220
3221static void cnic_ulp_start(struct cnic_dev *dev)
3222{
3223        struct cnic_local *cp = dev->cnic_priv;
3224        int if_type;
3225
3226        for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) {
3227                struct cnic_ulp_ops *ulp_ops;
3228
3229                mutex_lock(&cnic_lock);
3230                ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type],
3231                                                    lockdep_is_held(&cnic_lock));
3232                if (!ulp_ops || !ulp_ops->cnic_start) {
3233                        mutex_unlock(&cnic_lock);
3234                        continue;
3235                }
3236                set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
3237                mutex_unlock(&cnic_lock);
3238
3239                if (!test_and_set_bit(ULP_F_START, &cp->ulp_flags[if_type]))
3240                        ulp_ops->cnic_start(cp->ulp_handle[if_type]);
3241
3242                clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
3243        }
3244}
3245
3246static int cnic_copy_ulp_stats(struct cnic_dev *dev, int ulp_type)
3247{
3248        struct cnic_local *cp = dev->cnic_priv;
3249        struct cnic_ulp_ops *ulp_ops;
3250        int rc;
3251
3252        mutex_lock(&cnic_lock);
3253        ulp_ops = rcu_dereference_protected(cp->ulp_ops[ulp_type],
3254                                            lockdep_is_held(&cnic_lock));
3255        if (ulp_ops && ulp_ops->cnic_get_stats)
3256                rc = ulp_ops->cnic_get_stats(cp->ulp_handle[ulp_type]);
3257        else
3258                rc = -ENODEV;
3259        mutex_unlock(&cnic_lock);
3260        return rc;
3261}
3262
3263static int cnic_ctl(void *data, struct cnic_ctl_info *info)
3264{
3265        struct cnic_dev *dev = data;
3266        int ulp_type = CNIC_ULP_ISCSI;
3267
3268        switch (info->cmd) {
3269        case CNIC_CTL_STOP_CMD:
3270                cnic_hold(dev);
3271
3272                cnic_ulp_stop(dev);
3273                cnic_stop_hw(dev);
3274
3275                cnic_put(dev);
3276                break;
3277        case CNIC_CTL_START_CMD:
3278                cnic_hold(dev);
3279
3280                if (!cnic_start_hw(dev))
3281                        cnic_ulp_start(dev);
3282
3283                cnic_put(dev);
3284                break;
3285        case CNIC_CTL_STOP_ISCSI_CMD: {
3286                struct cnic_local *cp = dev->cnic_priv;
3287                set_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags);
3288                queue_delayed_work(cnic_wq, &cp->delete_task, 0);
3289                break;
3290        }
3291        case CNIC_CTL_COMPLETION_CMD: {
3292                struct cnic_ctl_completion *comp = &info->data.comp;
3293                u32 cid = BNX2X_SW_CID(comp->cid);
3294                u32 l5_cid;
3295                struct cnic_local *cp = dev->cnic_priv;
3296
3297                if (!test_bit(CNIC_F_CNIC_UP, &dev->flags))
3298                        break;
3299
3300                if (cnic_get_l5_cid(cp, cid, &l5_cid) == 0) {
3301                        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
3302
3303                        if (unlikely(comp->error)) {
3304                                set_bit(CTX_FL_CID_ERROR, &ctx->ctx_flags);
3305                                netdev_err(dev->netdev,
3306                                           "CID %x CFC delete comp error %x\n",
3307                                           cid, comp->error);
3308                        }
3309
3310                        ctx->wait_cond = 1;
3311                        wake_up(&ctx->waitq);
3312                }
3313                break;
3314        }
3315        case CNIC_CTL_FCOE_STATS_GET_CMD:
3316                ulp_type = CNIC_ULP_FCOE;
3317                /* fall through */
3318        case CNIC_CTL_ISCSI_STATS_GET_CMD:
3319                cnic_hold(dev);
3320                cnic_copy_ulp_stats(dev, ulp_type);
3321                cnic_put(dev);
3322                break;
3323
3324        default:
3325                return -EINVAL;
3326        }
3327        return 0;
3328}
3329
3330static void cnic_ulp_init(struct cnic_dev *dev)
3331{
3332        int i;
3333        struct cnic_local *cp = dev->cnic_priv;
3334
3335        for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) {
3336                struct cnic_ulp_ops *ulp_ops;
3337
3338                mutex_lock(&cnic_lock);
3339                ulp_ops = cnic_ulp_tbl_prot(i);
3340                if (!ulp_ops || !ulp_ops->cnic_init) {
3341                        mutex_unlock(&cnic_lock);
3342                        continue;
3343                }
3344                ulp_get(ulp_ops);
3345                mutex_unlock(&cnic_lock);
3346
3347                if (!test_and_set_bit(ULP_F_INIT, &cp->ulp_flags[i]))
3348                        ulp_ops->cnic_init(dev);
3349
3350                ulp_put(ulp_ops);
3351        }
3352}
3353
3354static void cnic_ulp_exit(struct cnic_dev *dev)
3355{
3356        int i;
3357        struct cnic_local *cp = dev->cnic_priv;
3358
3359        for (i = 0; i < MAX_CNIC_ULP_TYPE_EXT; i++) {
3360                struct cnic_ulp_ops *ulp_ops;
3361
3362                mutex_lock(&cnic_lock);
3363                ulp_ops = cnic_ulp_tbl_prot(i);
3364                if (!ulp_ops || !ulp_ops->cnic_exit) {
3365                        mutex_unlock(&cnic_lock);
3366                        continue;
3367                }
3368                ulp_get(ulp_ops);
3369                mutex_unlock(&cnic_lock);
3370
3371                if (test_and_clear_bit(ULP_F_INIT, &cp->ulp_flags[i]))
3372                        ulp_ops->cnic_exit(dev);
3373
3374                ulp_put(ulp_ops);
3375        }
3376}
3377
3378static int cnic_cm_offload_pg(struct cnic_sock *csk)
3379{
3380        struct cnic_dev *dev = csk->dev;
3381        struct l4_kwq_offload_pg *l4kwqe;
3382        struct kwqe *wqes[1];
3383
3384        l4kwqe = (struct l4_kwq_offload_pg *) &csk->kwqe1;
3385        memset(l4kwqe, 0, sizeof(*l4kwqe));
3386        wqes[0] = (struct kwqe *) l4kwqe;
3387
3388        l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_OFFLOAD_PG;
3389        l4kwqe->flags =
3390                L4_LAYER_CODE << L4_KWQ_OFFLOAD_PG_LAYER_CODE_SHIFT;
3391        l4kwqe->l2hdr_nbytes = ETH_HLEN;
3392
3393        l4kwqe->da0 = csk->ha[0];
3394        l4kwqe->da1 = csk->ha[1];
3395        l4kwqe->da2 = csk->ha[2];
3396        l4kwqe->da3 = csk->ha[3];
3397        l4kwqe->da4 = csk->ha[4];
3398        l4kwqe->da5 = csk->ha[5];
3399
3400        l4kwqe->sa0 = dev->mac_addr[0];
3401        l4kwqe->sa1 = dev->mac_addr[1];
3402        l4kwqe->sa2 = dev->mac_addr[2];
3403        l4kwqe->sa3 = dev->mac_addr[3];
3404        l4kwqe->sa4 = dev->mac_addr[4];
3405        l4kwqe->sa5 = dev->mac_addr[5];
3406
3407        l4kwqe->etype = ETH_P_IP;
3408        l4kwqe->ipid_start = DEF_IPID_START;
3409        l4kwqe->host_opaque = csk->l5_cid;
3410
3411        if (csk->vlan_id) {
3412                l4kwqe->pg_flags |= L4_KWQ_OFFLOAD_PG_VLAN_TAGGING;
3413                l4kwqe->vlan_tag = csk->vlan_id;
3414                l4kwqe->l2hdr_nbytes += 4;
3415        }
3416
3417        return dev->submit_kwqes(dev, wqes, 1);
3418}
3419
3420static int cnic_cm_update_pg(struct cnic_sock *csk)
3421{
3422        struct cnic_dev *dev = csk->dev;
3423        struct l4_kwq_update_pg *l4kwqe;
3424        struct kwqe *wqes[1];
3425
3426        l4kwqe = (struct l4_kwq_update_pg *) &csk->kwqe1;
3427        memset(l4kwqe, 0, sizeof(*l4kwqe));
3428        wqes[0] = (struct kwqe *) l4kwqe;
3429
3430        l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPDATE_PG;
3431        l4kwqe->flags =
3432                L4_LAYER_CODE << L4_KWQ_UPDATE_PG_LAYER_CODE_SHIFT;
3433        l4kwqe->pg_cid = csk->pg_cid;
3434
3435        l4kwqe->da0 = csk->ha[0];
3436        l4kwqe->da1 = csk->ha[1];
3437        l4kwqe->da2 = csk->ha[2];
3438        l4kwqe->da3 = csk->ha[3];
3439        l4kwqe->da4 = csk->ha[4];
3440        l4kwqe->da5 = csk->ha[5];
3441
3442        l4kwqe->pg_host_opaque = csk->l5_cid;
3443        l4kwqe->pg_valids = L4_KWQ_UPDATE_PG_VALIDS_DA;
3444
3445        return dev->submit_kwqes(dev, wqes, 1);
3446}
3447
3448static int cnic_cm_upload_pg(struct cnic_sock *csk)
3449{
3450        struct cnic_dev *dev = csk->dev;
3451        struct l4_kwq_upload *l4kwqe;
3452        struct kwqe *wqes[1];
3453
3454        l4kwqe = (struct l4_kwq_upload *) &csk->kwqe1;
3455        memset(l4kwqe, 0, sizeof(*l4kwqe));
3456        wqes[0] = (struct kwqe *) l4kwqe;
3457
3458        l4kwqe->opcode = L4_KWQE_OPCODE_VALUE_UPLOAD_PG;
3459        l4kwqe->flags =
3460                L4_LAYER_CODE << L4_KWQ_UPLOAD_LAYER_CODE_SHIFT;
3461        l4kwqe->cid = csk->pg_cid;
3462
3463        return dev->submit_kwqes(dev, wqes, 1);
3464}
3465
3466static int cnic_cm_conn_req(struct cnic_sock *csk)
3467{
3468        struct cnic_dev *dev = csk->dev;
3469        struct l4_kwq_connect_req1 *l4kwqe1;
3470        struct l4_kwq_connect_req2 *l4kwqe2;
3471        struct l4_kwq_connect_req3 *l4kwqe3;
3472        struct kwqe *wqes[3];
3473        u8 tcp_flags = 0;
3474        int num_wqes = 2;
3475
3476        l4kwqe1 = (struct l4_kwq_connect_req1 *) &csk->kwqe1;
3477        l4kwqe2 = (struct l4_kwq_connect_req2 *) &csk->kwqe2;
3478        l4kwqe3 = (struct l4_kwq_connect_req3 *) &csk->kwqe3;
3479        memset(l4kwqe1, 0, sizeof(*l4kwqe1));
3480        memset(l4kwqe2, 0, sizeof(*l4kwqe2));
3481        memset(l4kwqe3, 0, sizeof(*l4kwqe3));
3482
3483        l4kwqe3->op_code = L4_KWQE_OPCODE_VALUE_CONNECT3;
3484        l4kwqe3->flags =
3485                L4_LAYER_CODE << L4_KWQ_CONNECT_REQ3_LAYER_CODE_SHIFT;
3486        l4kwqe3->ka_timeout = csk->ka_timeout;
3487        l4kwqe3->ka_interval = csk->ka_interval;
3488        l4kwqe3->ka_max_probe_count = csk->ka_max_probe_count;
3489        l4kwqe3->tos = csk->tos;
3490        l4kwqe3->ttl = csk->ttl;
3491        l4kwqe3->snd_seq_scale = csk->snd_seq_scale;
3492        l4kwqe3->pmtu = csk->mtu;
3493        l4kwqe3->rcv_buf = csk->rcv_buf;
3494        l4kwqe3->snd_buf = csk->snd_buf;
3495        l4kwqe3->seed = csk->seed;
3496
3497        wqes[0] = (struct kwqe *) l4kwqe1;
3498        if (test_bit(SK_F_IPV6, &csk->flags)) {
3499                wqes[1] = (struct kwqe *) l4kwqe2;
3500                wqes[2] = (struct kwqe *) l4kwqe3;
3501                num_wqes = 3;
3502
3503                l4kwqe1->conn_flags = L4_KWQ_CONNECT_REQ1_IP_V6;
3504                l4kwqe2->op_code = L4_KWQE_OPCODE_VALUE_CONNECT2;
3505                l4kwqe2->flags =
3506                        L4_KWQ_CONNECT_REQ2_LINKED_WITH_NEXT |
3507                        L4_LAYER_CODE << L4_KWQ_CONNECT_REQ2_LAYER_CODE_SHIFT;
3508                l4kwqe2->src_ip_v6_2 = be32_to_cpu(csk->src_ip[1]);
3509                l4kwqe2->src_ip_v6_3 = be32_to_cpu(csk->src_ip[2]);
3510                l4kwqe2->src_ip_v6_4 = be32_to_cpu(csk->src_ip[3]);
3511                l4kwqe2->dst_ip_v6_2 = be32_to_cpu(csk->dst_ip[1]);
3512                l4kwqe2->dst_ip_v6_3 = be32_to_cpu(csk->dst_ip[2]);
3513                l4kwqe2->dst_ip_v6_4 = be32_to_cpu(csk->dst_ip[3]);
3514                l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct ipv6hdr) -
3515                               sizeof(struct tcphdr);
3516        } else {
3517                wqes[1] = (struct kwqe *) l4kwqe3;
3518                l4kwqe3->mss = l4kwqe3->pmtu - sizeof(struct iphdr) -
3519                               sizeof(struct tcphdr);
3520        }
3521
3522        l4kwqe1->op_code = L4_KWQE_OPCODE_VALUE_CONNECT1;
3523        l4kwqe1->flags =
3524                (L4_LAYER_CODE << L4_KWQ_CONNECT_REQ1_LAYER_CODE_SHIFT) |
3525                 L4_KWQ_CONNECT_REQ3_LINKED_WITH_NEXT;
3526        l4kwqe1->cid = csk->cid;
3527        l4kwqe1->pg_cid = csk->pg_cid;
3528        l4kwqe1->src_ip = be32_to_cpu(csk->src_ip[0]);
3529        l4kwqe1->dst_ip = be32_to_cpu(csk->dst_ip[0]);
3530        l4kwqe1->src_port = be16_to_cpu(csk->src_port);
3531        l4kwqe1->dst_port = be16_to_cpu(csk->dst_port);
3532        if (csk->tcp_flags & SK_TCP_NO_DELAY_ACK)
3533                tcp_flags |= L4_KWQ_CONNECT_REQ1_NO_DELAY_ACK;
3534        if (csk->tcp_flags & SK_TCP_KEEP_ALIVE)
3535                tcp_flags |= L4_KWQ_CONNECT_REQ1_KEEP_ALIVE;
3536        if (csk->tcp_flags & SK_TCP_NAGLE)
3537                tcp_flags |= L4_KWQ_CONNECT_REQ1_NAGLE_ENABLE;
3538        if (csk->tcp_flags & SK_TCP_TIMESTAMP)
3539                tcp_flags |= L4_KWQ_CONNECT_REQ1_TIME_STAMP;
3540        if (csk->tcp_flags & SK_TCP_SACK)
3541                tcp_flags |= L4_KWQ_CONNECT_REQ1_SACK;
3542        if (csk->tcp_flags & SK_TCP_SEG_SCALING)
3543                tcp_flags |= L4_KWQ_CONNECT_REQ1_SEG_SCALING;
3544
3545        l4kwqe1->tcp_flags = tcp_flags;
3546
3547        return dev->submit_kwqes(dev, wqes, num_wqes);
3548}
3549
3550static int cnic_cm_close_req(struct cnic_sock *csk)
3551{
3552        struct cnic_dev *dev = csk->dev;
3553        struct l4_kwq_close_req *l4kwqe;
3554        struct kwqe *wqes[1];
3555
3556        l4kwqe = (struct l4_kwq_close_req *) &csk->kwqe2;
3557        memset(l4kwqe, 0, sizeof(*l4kwqe));
3558        wqes[0] = (struct kwqe *) l4kwqe;
3559
3560        l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_CLOSE;
3561        l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_CLOSE_REQ_LAYER_CODE_SHIFT;
3562        l4kwqe->cid = csk->cid;
3563
3564        return dev->submit_kwqes(dev, wqes, 1);
3565}
3566
3567static int cnic_cm_abort_req(struct cnic_sock *csk)
3568{
3569        struct cnic_dev *dev = csk->dev;
3570        struct l4_kwq_reset_req *l4kwqe;
3571        struct kwqe *wqes[1];
3572
3573        l4kwqe = (struct l4_kwq_reset_req *) &csk->kwqe2;
3574        memset(l4kwqe, 0, sizeof(*l4kwqe));
3575        wqes[0] = (struct kwqe *) l4kwqe;
3576
3577        l4kwqe->op_code = L4_KWQE_OPCODE_VALUE_RESET;
3578        l4kwqe->flags = L4_LAYER_CODE << L4_KWQ_RESET_REQ_LAYER_CODE_SHIFT;
3579        l4kwqe->cid = csk->cid;
3580
3581        return dev->submit_kwqes(dev, wqes, 1);
3582}
3583
3584static int cnic_cm_create(struct cnic_dev *dev, int ulp_type, u32 cid,
3585                          u32 l5_cid, struct cnic_sock **csk, void *context)
3586{
3587        struct cnic_local *cp = dev->cnic_priv;
3588        struct cnic_sock *csk1;
3589
3590        if (l5_cid >= MAX_CM_SK_TBL_SZ)
3591                return -EINVAL;
3592
3593        if (cp->ctx_tbl) {
3594                struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
3595
3596                if (test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags))
3597                        return -EAGAIN;
3598        }
3599
3600        csk1 = &cp->csk_tbl[l5_cid];
3601        if (atomic_read(&csk1->ref_count))
3602                return -EAGAIN;
3603
3604        if (test_and_set_bit(SK_F_INUSE, &csk1->flags))
3605                return -EBUSY;
3606
3607        csk1->dev = dev;
3608        csk1->cid = cid;
3609        csk1->l5_cid = l5_cid;
3610        csk1->ulp_type = ulp_type;
3611        csk1->context = context;
3612
3613        csk1->ka_timeout = DEF_KA_TIMEOUT;
3614        csk1->ka_interval = DEF_KA_INTERVAL;
3615        csk1->ka_max_probe_count = DEF_KA_MAX_PROBE_COUNT;
3616        csk1->tos = DEF_TOS;
3617        csk1->ttl = DEF_TTL;
3618        csk1->snd_seq_scale = DEF_SND_SEQ_SCALE;
3619        csk1->rcv_buf = DEF_RCV_BUF;
3620        csk1->snd_buf = DEF_SND_BUF;
3621        csk1->seed = DEF_SEED;
3622        csk1->tcp_flags = 0;
3623
3624        *csk = csk1;
3625        return 0;
3626}
3627
3628static void cnic_cm_cleanup(struct cnic_sock *csk)
3629{
3630        if (csk->src_port) {
3631                struct cnic_dev *dev = csk->dev;
3632                struct cnic_local *cp = dev->cnic_priv;
3633
3634                cnic_free_id(&cp->csk_port_tbl, be16_to_cpu(csk->src_port));
3635                csk->src_port = 0;
3636        }
3637}
3638
3639static void cnic_close_conn(struct cnic_sock *csk)
3640{
3641        if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags)) {
3642                cnic_cm_upload_pg(csk);
3643                clear_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags);
3644        }
3645        cnic_cm_cleanup(csk);
3646}
3647
3648static int cnic_cm_destroy(struct cnic_sock *csk)
3649{
3650        if (!cnic_in_use(csk))
3651                return -EINVAL;
3652
3653        csk_hold(csk);
3654        clear_bit(SK_F_INUSE, &csk->flags);
3655        smp_mb__after_atomic();
3656        while (atomic_read(&csk->ref_count) != 1)
3657                msleep(1);
3658        cnic_cm_cleanup(csk);
3659
3660        csk->flags = 0;
3661        csk_put(csk);
3662        return 0;
3663}
3664
3665static inline u16 cnic_get_vlan(struct net_device *dev,
3666                                struct net_device **vlan_dev)
3667{
3668        if (dev->priv_flags & IFF_802_1Q_VLAN) {
3669                *vlan_dev = vlan_dev_real_dev(dev);
3670                return vlan_dev_vlan_id(dev);
3671        }
3672        *vlan_dev = dev;
3673        return 0;
3674}
3675
3676static int cnic_get_v4_route(struct sockaddr_in *dst_addr,
3677                             struct dst_entry **dst)
3678{
3679#if defined(CONFIG_INET)
3680        struct rtable *rt;
3681
3682        rt = ip_route_output(&init_net, dst_addr->sin_addr.s_addr, 0, 0, 0);
3683        if (!IS_ERR(rt)) {
3684                *dst = &rt->dst;
3685                return 0;
3686        }
3687        return PTR_ERR(rt);
3688#else
3689        return -ENETUNREACH;
3690#endif
3691}
3692
3693static int cnic_get_v6_route(struct sockaddr_in6 *dst_addr,
3694                             struct dst_entry **dst)
3695{
3696#if IS_ENABLED(CONFIG_IPV6)
3697        struct flowi6 fl6;
3698
3699        memset(&fl6, 0, sizeof(fl6));
3700        fl6.daddr = dst_addr->sin6_addr;
3701        if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
3702                fl6.flowi6_oif = dst_addr->sin6_scope_id;
3703
3704        *dst = ip6_route_output(&init_net, NULL, &fl6);
3705        if ((*dst)->error) {
3706                dst_release(*dst);
3707                *dst = NULL;
3708                return -ENETUNREACH;
3709        } else
3710                return 0;
3711#endif
3712
3713        return -ENETUNREACH;
3714}
3715
3716static struct cnic_dev *cnic_cm_select_dev(struct sockaddr_in *dst_addr,
3717                                           int ulp_type)
3718{
3719        struct cnic_dev *dev = NULL;
3720        struct dst_entry *dst;
3721        struct net_device *netdev = NULL;
3722        int err = -ENETUNREACH;
3723
3724        if (dst_addr->sin_family == AF_INET)
3725                err = cnic_get_v4_route(dst_addr, &dst);
3726        else if (dst_addr->sin_family == AF_INET6) {
3727                struct sockaddr_in6 *dst_addr6 =
3728                        (struct sockaddr_in6 *) dst_addr;
3729
3730                err = cnic_get_v6_route(dst_addr6, &dst);
3731        } else
3732                return NULL;
3733
3734        if (err)
3735                return NULL;
3736
3737        if (!dst->dev)
3738                goto done;
3739
3740        cnic_get_vlan(dst->dev, &netdev);
3741
3742        dev = cnic_from_netdev(netdev);
3743
3744done:
3745        dst_release(dst);
3746        if (dev)
3747                cnic_put(dev);
3748        return dev;
3749}
3750
3751static int cnic_resolve_addr(struct cnic_sock *csk, struct cnic_sockaddr *saddr)
3752{
3753        struct cnic_dev *dev = csk->dev;
3754        struct cnic_local *cp = dev->cnic_priv;
3755
3756        return cnic_send_nlmsg(cp, ISCSI_KEVENT_PATH_REQ, csk);
3757}
3758
3759static int cnic_get_route(struct cnic_sock *csk, struct cnic_sockaddr *saddr)
3760{
3761        struct cnic_dev *dev = csk->dev;
3762        struct cnic_local *cp = dev->cnic_priv;
3763        int is_v6, rc = 0;
3764        struct dst_entry *dst = NULL;
3765        struct net_device *realdev;
3766        __be16 local_port;
3767        u32 port_id;
3768
3769        if (saddr->local.v6.sin6_family == AF_INET6 &&
3770            saddr->remote.v6.sin6_family == AF_INET6)
3771                is_v6 = 1;
3772        else if (saddr->local.v4.sin_family == AF_INET &&
3773                 saddr->remote.v4.sin_family == AF_INET)
3774                is_v6 = 0;
3775        else
3776                return -EINVAL;
3777
3778        clear_bit(SK_F_IPV6, &csk->flags);
3779
3780        if (is_v6) {
3781                set_bit(SK_F_IPV6, &csk->flags);
3782                cnic_get_v6_route(&saddr->remote.v6, &dst);
3783
3784                memcpy(&csk->dst_ip[0], &saddr->remote.v6.sin6_addr,
3785                       sizeof(struct in6_addr));
3786                csk->dst_port = saddr->remote.v6.sin6_port;
3787                local_port = saddr->local.v6.sin6_port;
3788
3789        } else {
3790                cnic_get_v4_route(&saddr->remote.v4, &dst);
3791
3792                csk->dst_ip[0] = saddr->remote.v4.sin_addr.s_addr;
3793                csk->dst_port = saddr->remote.v4.sin_port;
3794                local_port = saddr->local.v4.sin_port;
3795        }
3796
3797        csk->vlan_id = 0;
3798        csk->mtu = dev->netdev->mtu;
3799        if (dst && dst->dev) {
3800                u16 vlan = cnic_get_vlan(dst->dev, &realdev);
3801                if (realdev == dev->netdev) {
3802                        csk->vlan_id = vlan;
3803                        csk->mtu = dst_mtu(dst);
3804                }
3805        }
3806
3807        port_id = be16_to_cpu(local_port);
3808        if (port_id >= CNIC_LOCAL_PORT_MIN &&
3809            port_id < CNIC_LOCAL_PORT_MAX) {
3810                if (cnic_alloc_id(&cp->csk_port_tbl, port_id))
3811                        port_id = 0;
3812        } else
3813                port_id = 0;
3814
3815        if (!port_id) {
3816                port_id = cnic_alloc_new_id(&cp->csk_port_tbl);
3817                if (port_id == -1) {
3818                        rc = -ENOMEM;
3819                        goto err_out;
3820                }
3821                local_port = cpu_to_be16(port_id);
3822        }
3823        csk->src_port = local_port;
3824
3825err_out:
3826        dst_release(dst);
3827        return rc;
3828}
3829
3830static void cnic_init_csk_state(struct cnic_sock *csk)
3831{
3832        csk->state = 0;
3833        clear_bit(SK_F_OFFLD_SCHED, &csk->flags);
3834        clear_bit(SK_F_CLOSING, &csk->flags);
3835}
3836
3837static int cnic_cm_connect(struct cnic_sock *csk, struct cnic_sockaddr *saddr)
3838{
3839        struct cnic_local *cp = csk->dev->cnic_priv;
3840        int err = 0;
3841
3842        if (cp->ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)
3843                return -EOPNOTSUPP;
3844
3845        if (!cnic_in_use(csk))
3846                return -EINVAL;
3847
3848        if (test_and_set_bit(SK_F_CONNECT_START, &csk->flags))
3849                return -EINVAL;
3850
3851        cnic_init_csk_state(csk);
3852
3853        err = cnic_get_route(csk, saddr);
3854        if (err)
3855                goto err_out;
3856
3857        err = cnic_resolve_addr(csk, saddr);
3858        if (!err)
3859                return 0;
3860
3861err_out:
3862        clear_bit(SK_F_CONNECT_START, &csk->flags);
3863        return err;
3864}
3865
3866static int cnic_cm_abort(struct cnic_sock *csk)
3867{
3868        struct cnic_local *cp = csk->dev->cnic_priv;
3869        u32 opcode = L4_KCQE_OPCODE_VALUE_RESET_COMP;
3870
3871        if (!cnic_in_use(csk))
3872                return -EINVAL;
3873
3874        if (cnic_abort_prep(csk))
3875                return cnic_cm_abort_req(csk);
3876
3877        /* Getting here means that we haven't started connect, or
3878         * connect was not successful, or it has been reset by the target.
3879         */
3880
3881        cp->close_conn(csk, opcode);
3882        if (csk->state != opcode) {
3883                /* Wait for remote reset sequence to complete */
3884                while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags))
3885                        msleep(1);
3886
3887                return -EALREADY;
3888        }
3889
3890        return 0;
3891}
3892
3893static int cnic_cm_close(struct cnic_sock *csk)
3894{
3895        if (!cnic_in_use(csk))
3896                return -EINVAL;
3897
3898        if (cnic_close_prep(csk)) {
3899                csk->state = L4_KCQE_OPCODE_VALUE_CLOSE_COMP;
3900                return cnic_cm_close_req(csk);
3901        } else {
3902                /* Wait for remote reset sequence to complete */
3903                while (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags))
3904                        msleep(1);
3905
3906                return -EALREADY;
3907        }
3908        return 0;
3909}
3910
3911static void cnic_cm_upcall(struct cnic_local *cp, struct cnic_sock *csk,
3912                           u8 opcode)
3913{
3914        struct cnic_ulp_ops *ulp_ops;
3915        int ulp_type = csk->ulp_type;
3916
3917        rcu_read_lock();
3918        ulp_ops = rcu_dereference(cp->ulp_ops[ulp_type]);
3919        if (ulp_ops) {
3920                if (opcode == L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE)
3921                        ulp_ops->cm_connect_complete(csk);
3922                else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_COMP)
3923                        ulp_ops->cm_close_complete(csk);
3924                else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED)
3925                        ulp_ops->cm_remote_abort(csk);
3926                else if (opcode == L4_KCQE_OPCODE_VALUE_RESET_COMP)
3927                        ulp_ops->cm_abort_complete(csk);
3928                else if (opcode == L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED)
3929                        ulp_ops->cm_remote_close(csk);
3930        }
3931        rcu_read_unlock();
3932}
3933
3934static int cnic_cm_set_pg(struct cnic_sock *csk)
3935{
3936        if (cnic_offld_prep(csk)) {
3937                if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags))
3938                        cnic_cm_update_pg(csk);
3939                else
3940                        cnic_cm_offload_pg(csk);
3941        }
3942        return 0;
3943}
3944
3945static void cnic_cm_process_offld_pg(struct cnic_dev *dev, struct l4_kcq *kcqe)
3946{
3947        struct cnic_local *cp = dev->cnic_priv;
3948        u32 l5_cid = kcqe->pg_host_opaque;
3949        u8 opcode = kcqe->op_code;
3950        struct cnic_sock *csk = &cp->csk_tbl[l5_cid];
3951
3952        csk_hold(csk);
3953        if (!cnic_in_use(csk))
3954                goto done;
3955
3956        if (opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) {
3957                clear_bit(SK_F_OFFLD_SCHED, &csk->flags);
3958                goto done;
3959        }
3960        /* Possible PG kcqe status:  SUCCESS, OFFLOADED_PG, or CTX_ALLOC_FAIL */
3961        if (kcqe->status == L4_KCQE_COMPLETION_STATUS_CTX_ALLOC_FAIL) {
3962                clear_bit(SK_F_OFFLD_SCHED, &csk->flags);
3963                cnic_cm_upcall(cp, csk,
3964                               L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE);
3965                goto done;
3966        }
3967
3968        csk->pg_cid = kcqe->pg_cid;
3969        set_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags);
3970        cnic_cm_conn_req(csk);
3971
3972done:
3973        csk_put(csk);
3974}
3975
3976static void cnic_process_fcoe_term_conn(struct cnic_dev *dev, struct kcqe *kcqe)
3977{
3978        struct cnic_local *cp = dev->cnic_priv;
3979        struct fcoe_kcqe *fc_kcqe = (struct fcoe_kcqe *) kcqe;
3980        u32 l5_cid = fc_kcqe->fcoe_conn_id + BNX2X_FCOE_L5_CID_BASE;
3981        struct cnic_context *ctx = &cp->ctx_tbl[l5_cid];
3982
3983        ctx->timestamp = jiffies;
3984        ctx->wait_cond = 1;
3985        wake_up(&ctx->waitq);
3986}
3987
3988static void cnic_cm_process_kcqe(struct cnic_dev *dev, struct kcqe *kcqe)
3989{
3990        struct cnic_local *cp = dev->cnic_priv;
3991        struct l4_kcq *l4kcqe = (struct l4_kcq *) kcqe;
3992        u8 opcode = l4kcqe->op_code;
3993        u32 l5_cid;
3994        struct cnic_sock *csk;
3995
3996        if (opcode == FCOE_RAMROD_CMD_ID_TERMINATE_CONN) {
3997                cnic_process_fcoe_term_conn(dev, kcqe);
3998                return;
3999        }
4000        if (opcode == L4_KCQE_OPCODE_VALUE_OFFLOAD_PG ||
4001            opcode == L4_KCQE_OPCODE_VALUE_UPDATE_PG) {
4002                cnic_cm_process_offld_pg(dev, l4kcqe);
4003                return;
4004        }
4005
4006        l5_cid = l4kcqe->conn_id;
4007        if (opcode & 0x80)
4008                l5_cid = l4kcqe->cid;
4009        if (l5_cid >= MAX_CM_SK_TBL_SZ)
4010                return;
4011
4012        csk = &cp->csk_tbl[l5_cid];
4013        csk_hold(csk);
4014
4015        if (!cnic_in_use(csk)) {
4016                csk_put(csk);
4017                return;
4018        }
4019
4020        switch (opcode) {
4021        case L5CM_RAMROD_CMD_ID_TCP_CONNECT:
4022                if (l4kcqe->status != 0) {
4023                        clear_bit(SK_F_OFFLD_SCHED, &csk->flags);
4024                        cnic_cm_upcall(cp, csk,
4025                                       L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE);
4026                }
4027                break;
4028        case L4_KCQE_OPCODE_VALUE_CONNECT_COMPLETE:
4029                if (l4kcqe->status == 0)
4030                        set_bit(SK_F_OFFLD_COMPLETE, &csk->flags);
4031                else if (l4kcqe->status ==
4032                         L4_KCQE_COMPLETION_STATUS_PARITY_ERROR)
4033                        set_bit(SK_F_HW_ERR, &csk->flags);
4034
4035                smp_mb__before_atomic();
4036                clear_bit(SK_F_OFFLD_SCHED, &csk->flags);
4037                cnic_cm_upcall(cp, csk, opcode);
4038                break;
4039
4040        case L5CM_RAMROD_CMD_ID_CLOSE: {
4041                struct iscsi_kcqe *l5kcqe = (struct iscsi_kcqe *) kcqe;
4042
4043                if (l4kcqe->status != 0 || l5kcqe->completion_status != 0) {
4044                        netdev_warn(dev->netdev, "RAMROD CLOSE compl with status 0x%x completion status 0x%x\n",
4045                                    l4kcqe->status, l5kcqe->completion_status);
4046                        opcode = L4_KCQE_OPCODE_VALUE_CLOSE_COMP;
4047                        /* Fall through */
4048                } else {
4049                        break;
4050                }
4051        }
4052        case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED:
4053        case L4_KCQE_OPCODE_VALUE_CLOSE_COMP:
4054        case L4_KCQE_OPCODE_VALUE_RESET_COMP:
4055        case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE:
4056        case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD:
4057                if (l4kcqe->status == L4_KCQE_COMPLETION_STATUS_PARITY_ERROR)
4058                        set_bit(SK_F_HW_ERR, &csk->flags);
4059
4060                cp->close_conn(csk, opcode);
4061                break;
4062
4063        case L4_KCQE_OPCODE_VALUE_CLOSE_RECEIVED:
4064                /* after we already sent CLOSE_REQ */
4065                if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags) &&
4066                    !test_bit(SK_F_OFFLD_COMPLETE, &csk->flags) &&
4067                    csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP)
4068                        cp->close_conn(csk, L4_KCQE_OPCODE_VALUE_RESET_COMP);
4069                else
4070                        cnic_cm_upcall(cp, csk, opcode);
4071                break;
4072        }
4073        csk_put(csk);
4074}
4075
4076static void cnic_cm_indicate_kcqe(void *data, struct kcqe *kcqe[], u32 num)
4077{
4078        struct cnic_dev *dev = data;
4079        int i;
4080
4081        for (i = 0; i < num; i++)
4082                cnic_cm_process_kcqe(dev, kcqe[i]);
4083}
4084
4085static struct cnic_ulp_ops cm_ulp_ops = {
4086        .indicate_kcqes         = cnic_cm_indicate_kcqe,
4087};
4088
4089static void cnic_cm_free_mem(struct cnic_dev *dev)
4090{
4091        struct cnic_local *cp = dev->cnic_priv;
4092
4093        kfree(cp->csk_tbl);
4094        cp->csk_tbl = NULL;
4095        cnic_free_id_tbl(&cp->csk_port_tbl);
4096}
4097
4098static int cnic_cm_alloc_mem(struct cnic_dev *dev)
4099{
4100        struct cnic_local *cp = dev->cnic_priv;
4101        u32 port_id;
4102
4103        cp->csk_tbl = kzalloc(sizeof(struct cnic_sock) * MAX_CM_SK_TBL_SZ,
4104                              GFP_KERNEL);
4105        if (!cp->csk_tbl)
4106                return -ENOMEM;
4107
4108        port_id = prandom_u32();
4109        port_id %= CNIC_LOCAL_PORT_RANGE;
4110        if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE,
4111                             CNIC_LOCAL_PORT_MIN, port_id)) {
4112                cnic_cm_free_mem(dev);
4113                return -ENOMEM;
4114        }
4115        return 0;
4116}
4117
4118static int cnic_ready_to_close(struct cnic_sock *csk, u32 opcode)
4119{
4120        if (test_and_clear_bit(SK_F_OFFLD_COMPLETE, &csk->flags)) {
4121                /* Unsolicited RESET_COMP or RESET_RECEIVED */
4122                opcode = L4_KCQE_OPCODE_VALUE_RESET_RECEIVED;
4123                csk->state = opcode;
4124        }
4125
4126        /* 1. If event opcode matches the expected event in csk->state
4127         * 2. If the expected event is CLOSE_COMP or RESET_COMP, we accept any
4128         *    event
4129         * 3. If the expected event is 0, meaning the connection was never
4130         *    never established, we accept the opcode from cm_abort.
4131         */
4132        if (opcode == csk->state || csk->state == 0 ||
4133            csk->state == L4_KCQE_OPCODE_VALUE_CLOSE_COMP ||
4134            csk->state == L4_KCQE_OPCODE_VALUE_RESET_COMP) {
4135                if (!test_and_set_bit(SK_F_CLOSING, &csk->flags)) {
4136                        if (csk->state == 0)
4137                                csk->state = opcode;
4138                        return 1;
4139                }
4140        }
4141        return 0;
4142}
4143
4144static void cnic_close_bnx2_conn(struct cnic_sock *csk, u32 opcode)
4145{
4146        struct cnic_dev *dev = csk->dev;
4147        struct cnic_local *cp = dev->cnic_priv;
4148
4149        if (opcode == L4_KCQE_OPCODE_VALUE_RESET_RECEIVED) {
4150                cnic_cm_upcall(cp, csk, opcode);
4151                return;
4152        }
4153
4154        clear_bit(SK_F_CONNECT_START, &csk->flags);
4155        cnic_close_conn(csk);
4156        csk->state = opcode;
4157        cnic_cm_upcall(cp, csk, opcode);
4158}
4159
4160static void cnic_cm_stop_bnx2_hw(struct cnic_dev *dev)
4161{
4162}
4163
4164static int cnic_cm_init_bnx2_hw(struct cnic_dev *dev)
4165{
4166        u32 seed;
4167
4168        seed = prandom_u32();
4169        cnic_ctx_wr(dev, 45, 0, seed);
4170        return 0;
4171}
4172
4173static void cnic_close_bnx2x_conn(struct cnic_sock *csk, u32 opcode)
4174{
4175        struct cnic_dev *dev = csk->dev;
4176        struct cnic_local *cp = dev->cnic_priv;
4177        struct cnic_context *ctx = &cp->ctx_tbl[csk->l5_cid];
4178        union l5cm_specific_data l5_data;
4179        u32 cmd = 0;
4180        int close_complete = 0;
4181
4182        switch (opcode) {
4183        case L4_KCQE_OPCODE_VALUE_RESET_RECEIVED:
4184        case L4_KCQE_OPCODE_VALUE_CLOSE_COMP:
4185        case L4_KCQE_OPCODE_VALUE_RESET_COMP:
4186                if (cnic_ready_to_close(csk, opcode)) {
4187                        if (test_bit(SK_F_HW_ERR, &csk->flags))
4188                                close_complete = 1;
4189                        else if (test_bit(SK_F_PG_OFFLD_COMPLETE, &csk->flags))
4190                                cmd = L5CM_RAMROD_CMD_ID_SEARCHER_DELETE;
4191                        else
4192                                close_complete = 1;
4193                }
4194                break;
4195        case L5CM_RAMROD_CMD_ID_SEARCHER_DELETE:
4196                cmd = L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD;
4197                break;
4198        case L5CM_RAMROD_CMD_ID_TERMINATE_OFFLOAD:
4199                close_complete = 1;
4200                break;
4201        }
4202        if (cmd) {
4203                memset(&l5_data, 0, sizeof(l5_data));
4204
4205                cnic_submit_kwqe_16(dev, cmd, csk->cid, ISCSI_CONNECTION_TYPE,
4206                                    &l5_data);
4207        } else if (close_complete) {
4208                ctx->timestamp = jiffies;
4209                cnic_close_conn(csk);
4210                cnic_cm_upcall(cp, csk, csk->state);
4211        }
4212}
4213
4214static void cnic_cm_stop_bnx2x_hw(struct cnic_dev *dev)
4215{
4216        struct cnic_local *cp = dev->cnic_priv;
4217
4218        if (!cp->ctx_tbl)
4219                return;
4220
4221        if (!netif_running(dev->netdev))
4222                return;
4223
4224        cnic_bnx2x_delete_wait(dev, 0);
4225
4226        cancel_delayed_work(&cp->delete_task);
4227        flush_workqueue(cnic_wq);
4228
4229        if (atomic_read(&cp->iscsi_conn) != 0)
4230                netdev_warn(dev->netdev, "%d iSCSI connections not destroyed\n",
4231                            atomic_read(&cp->iscsi_conn));
4232}
4233
4234static int cnic_cm_init_bnx2x_hw(struct cnic_dev *dev)
4235{
4236        struct bnx2x *bp = netdev_priv(dev->netdev);
4237        u32 pfid = bp->pfid;
4238        u32 port = BP_PORT(bp);
4239
4240        cnic_init_bnx2x_mac(dev);
4241        cnic_bnx2x_set_tcp_options(dev, 0, 1);
4242
4243        CNIC_WR16(dev, BAR_XSTRORM_INTMEM +
4244                  XSTORM_ISCSI_LOCAL_VLAN_OFFSET(pfid), 0);
4245
4246        CNIC_WR(dev, BAR_XSTRORM_INTMEM +
4247                XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_ENABLED_OFFSET(port), 1);
4248        CNIC_WR(dev, BAR_XSTRORM_INTMEM +
4249                XSTORM_TCP_GLOBAL_DEL_ACK_COUNTER_MAX_COUNT_OFFSET(port),
4250                DEF_MAX_DA_COUNT);
4251
4252        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
4253                 XSTORM_ISCSI_TCP_VARS_TTL_OFFSET(pfid), DEF_TTL);
4254        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
4255                 XSTORM_ISCSI_TCP_VARS_TOS_OFFSET(pfid), DEF_TOS);
4256        CNIC_WR8(dev, BAR_XSTRORM_INTMEM +
4257                 XSTORM_ISCSI_TCP_VARS_ADV_WND_SCL_OFFSET(pfid), 2);
4258        CNIC_WR(dev, BAR_XSTRORM_INTMEM +
4259                XSTORM_TCP_TX_SWS_TIMER_VAL_OFFSET(pfid), DEF_SWS_TIMER);
4260
4261        CNIC_WR(dev, BAR_TSTRORM_INTMEM + TSTORM_TCP_MAX_CWND_OFFSET(pfid),
4262                DEF_MAX_CWND);
4263        return 0;
4264}
4265
4266static void cnic_delete_task(struct work_struct *work)
4267{
4268        struct cnic_local *cp;
4269        struct cnic_dev *dev;
4270        u32 i;
4271        int need_resched = 0;
4272
4273        cp = container_of(work, struct cnic_local, delete_task.work);
4274        dev = cp->dev;
4275
4276        if (test_and_clear_bit(CNIC_LCL_FL_STOP_ISCSI, &cp->cnic_local_flags)) {
4277                struct drv_ctl_info info;
4278
4279                cnic_ulp_stop_one(cp, CNIC_ULP_ISCSI);
4280
4281                memset(&info, 0, sizeof(struct drv_ctl_info));
4282                info.cmd = DRV_CTL_ISCSI_STOPPED_CMD;
4283                cp->ethdev->drv_ctl(dev->netdev, &info);
4284        }
4285
4286        for (i = 0; i < cp->max_cid_space; i++) {
4287                struct cnic_context *ctx = &cp->ctx_tbl[i];
4288                int err;
4289
4290                if (!test_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags) ||
4291                    !test_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags))
4292                        continue;
4293
4294                if (!time_after(jiffies, ctx->timestamp + (2 * HZ))) {
4295                        need_resched = 1;
4296                        continue;
4297                }
4298
4299                if (!test_and_clear_bit(CTX_FL_DELETE_WAIT, &ctx->ctx_flags))
4300                        continue;
4301
4302                err = cnic_bnx2x_destroy_ramrod(dev, i);
4303
4304                cnic_free_bnx2x_conn_resc(dev, i);
4305                if (!err) {
4306                        if (ctx->ulp_proto_id == CNIC_ULP_ISCSI)
4307                                atomic_dec(&cp->iscsi_conn);
4308
4309                        clear_bit(CTX_FL_OFFLD_START, &ctx->ctx_flags);
4310                }
4311        }
4312
4313        if (need_resched)
4314                queue_delayed_work(cnic_wq, &cp->delete_task,
4315                                   msecs_to_jiffies(10));
4316
4317}
4318
4319static int cnic_cm_open(struct cnic_dev *dev)
4320{
4321        struct cnic_local *cp = dev->cnic_priv;
4322        int err;
4323
4324        err = cnic_cm_alloc_mem(dev);
4325        if (err)
4326                return err;
4327
4328        err = cp->start_cm(dev);
4329
4330        if (err)
4331                goto err_out;
4332
4333        INIT_DELAYED_WORK(&cp->delete_task, cnic_delete_task);
4334
4335        dev->cm_create = cnic_cm_create;
4336        dev->cm_destroy = cnic_cm_destroy;
4337        dev->cm_connect = cnic_cm_connect;
4338        dev->cm_abort = cnic_cm_abort;
4339        dev->cm_close = cnic_cm_close;
4340        dev->cm_select_dev = cnic_cm_select_dev;
4341
4342        cp->ulp_handle[CNIC_ULP_L4] = dev;
4343        rcu_assign_pointer(cp->ulp_ops[CNIC_ULP_L4], &cm_ulp_ops);
4344        return 0;
4345
4346err_out:
4347        cnic_cm_free_mem(dev);
4348        return err;
4349}
4350
4351static int cnic_cm_shutdown(struct cnic_dev *dev)
4352{
4353        struct cnic_local *cp = dev->cnic_priv;
4354        int i;
4355
4356        if (!cp->csk_tbl)
4357                return 0;
4358
4359        for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) {
4360                struct cnic_sock *csk = &cp->csk_tbl[i];
4361
4362                clear_bit(SK_F_INUSE, &csk->flags);
4363                cnic_cm_cleanup(csk);
4364        }
4365        cnic_cm_free_mem(dev);
4366
4367        return 0;
4368}
4369
4370static void cnic_init_context(struct cnic_dev *dev, u32 cid)
4371{
4372        u32 cid_addr;
4373        int i;
4374
4375        cid_addr = GET_CID_ADDR(cid);
4376
4377        for (i = 0; i < CTX_SIZE; i += 4)
4378                cnic_ctx_wr(dev, cid_addr, i, 0);
4379}
4380
4381static int cnic_setup_5709_context(struct cnic_dev *dev, int valid)
4382{
4383        struct cnic_local *cp = dev->cnic_priv;
4384        int ret = 0, i;
4385        u32 valid_bit = valid ? BNX2_CTX_HOST_PAGE_TBL_DATA0_VALID : 0;
4386
4387        if (BNX2_CHIP(cp) != BNX2_CHIP_5709)
4388                return 0;
4389
4390        for (i = 0; i < cp->ctx_blks; i++) {
4391                int j;
4392                u32 idx = cp->ctx_arr[i].cid / cp->cids_per_blk;
4393                u32 val;
4394
4395                memset(cp->ctx_arr[i].ctx, 0, CNIC_PAGE_SIZE);
4396
4397                CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA0,
4398                        (cp->ctx_arr[i].mapping & 0xffffffff) | valid_bit);
4399                CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_DATA1,
4400                        (u64) cp->ctx_arr[i].mapping >> 32);
4401                CNIC_WR(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL, idx |
4402                        BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ);
4403                for (j = 0; j < 10; j++) {
4404
4405                        val = CNIC_RD(dev, BNX2_CTX_HOST_PAGE_TBL_CTRL);
4406                        if (!(val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ))
4407                                break;
4408                        udelay(5);
4409                }
4410                if (val & BNX2_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) {
4411                        ret = -EBUSY;
4412                        break;
4413                }
4414        }
4415        return ret;
4416}
4417
4418static void cnic_free_irq(struct cnic_dev *dev)
4419{
4420        struct cnic_local *cp = dev->cnic_priv;
4421        struct cnic_eth_dev *ethdev = cp->ethdev;
4422
4423        if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) {
4424                cp->disable_int_sync(dev);
4425                tasklet_kill(&cp->cnic_irq_task);
4426                free_irq(ethdev->irq_arr[0].vector, dev);
4427        }
4428}
4429
4430static int cnic_request_irq(struct cnic_dev *dev)
4431{
4432        struct cnic_local *cp = dev->cnic_priv;
4433        struct cnic_eth_dev *ethdev = cp->ethdev;
4434        int err;
4435
4436        err = request_irq(ethdev->irq_arr[0].vector, cnic_irq, 0, "cnic", dev);
4437        if (err)
4438                tasklet_disable(&cp->cnic_irq_task);
4439
4440        return err;
4441}
4442
4443static int cnic_init_bnx2_irq(struct cnic_dev *dev)
4444{
4445        struct cnic_local *cp = dev->cnic_priv;
4446        struct cnic_eth_dev *ethdev = cp->ethdev;
4447
4448        if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) {
4449                int err, i = 0;
4450                int sblk_num = cp->status_blk_num;
4451                u32 base = ((sblk_num - 1) * BNX2_HC_SB_CONFIG_SIZE) +
4452                           BNX2_HC_SB_CONFIG_1;
4453
4454                CNIC_WR(dev, base, BNX2_HC_SB_CONFIG_1_ONE_SHOT);
4455
4456                CNIC_WR(dev, base + BNX2_HC_COMP_PROD_TRIP_OFF, (2 << 16) | 8);
4457                CNIC_WR(dev, base + BNX2_HC_COM_TICKS_OFF, (64 << 16) | 220);
4458                CNIC_WR(dev, base + BNX2_HC_CMD_TICKS_OFF, (64 << 16) | 220);
4459
4460                cp->last_status_idx = cp->status_blk.bnx2->status_idx;
4461                tasklet_init(&cp->cnic_irq_task, cnic_service_bnx2_msix,
4462                             (unsigned long) dev);
4463                err = cnic_request_irq(dev);
4464                if (err)
4465                        return err;
4466
4467                while (cp->status_blk.bnx2->status_completion_producer_index &&
4468                       i < 10) {
4469                        CNIC_WR(dev, BNX2_HC_COALESCE_NOW,
4470                                1 << (11 + sblk_num));
4471                        udelay(10);
4472                        i++;
4473                        barrier();
4474                }
4475                if (cp->status_blk.bnx2->status_completion_producer_index) {
4476                        cnic_free_irq(dev);
4477                        goto failed;
4478                }
4479
4480        } else {
4481                struct status_block *sblk = cp->status_blk.gen;
4482                u32 hc_cmd = CNIC_RD(dev, BNX2_HC_COMMAND);
4483                int i = 0;
4484
4485                while (sblk->status_completion_producer_index && i < 10) {
4486                        CNIC_WR(dev, BNX2_HC_COMMAND,
4487                                hc_cmd | BNX2_HC_COMMAND_COAL_NOW_WO_INT);
4488                        udelay(10);
4489                        i++;
4490                        barrier();
4491                }
4492                if (sblk->status_completion_producer_index)
4493                        goto failed;
4494
4495        }
4496        return 0;
4497
4498failed:
4499        netdev_err(dev->netdev, "KCQ index not resetting to 0\n");
4500        return -EBUSY;
4501}
4502
4503static void cnic_enable_bnx2_int(struct cnic_dev *dev)
4504{
4505        struct cnic_local *cp = dev->cnic_priv;
4506        struct cnic_eth_dev *ethdev = cp->ethdev;
4507
4508        if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX))
4509                return;
4510
4511        CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num |
4512                BNX2_PCICFG_INT_ACK_CMD_INDEX_VALID | cp->last_status_idx);
4513}
4514
4515static void cnic_disable_bnx2_int_sync(struct cnic_dev *dev)
4516{
4517        struct cnic_local *cp = dev->cnic_priv;
4518        struct cnic_eth_dev *ethdev = cp->ethdev;
4519
4520        if (!(ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX))
4521                return;
4522
4523        CNIC_WR(dev, BNX2_PCICFG_INT_ACK_CMD, cp->int_num |
4524                BNX2_PCICFG_INT_ACK_CMD_MASK_INT);
4525        CNIC_RD(dev, BNX2_PCICFG_INT_ACK_CMD);
4526        synchronize_irq(ethdev->irq_arr[0].vector);
4527}
4528
4529static void cnic_init_bnx2_tx_ring(struct cnic_dev *dev)
4530{
4531        struct cnic_local *cp = dev->cnic_priv;
4532        struct cnic_eth_dev *ethdev = cp->ethdev;
4533        struct cnic_uio_dev *udev = cp->udev;
4534        u32 cid_addr, tx_cid, sb_id;
4535        u32 val, offset0, offset1, offset2, offset3;
4536        int i;
4537        struct bnx2_tx_bd *txbd;
4538        dma_addr_t buf_map, ring_map = udev->l2_ring_map;
4539        struct status_block *s_blk = cp->status_blk.gen;
4540
4541        sb_id = cp->status_blk_num;
4542        tx_cid = 20;
4543        cp->tx_cons_ptr = &s_blk->status_tx_quick_consumer_index2;
4544        if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) {
4545                struct status_block_msix *sblk = cp->status_blk.bnx2;
4546
4547                tx_cid = TX_TSS_CID + sb_id - 1;
4548                CNIC_WR(dev, BNX2_TSCH_TSS_CFG, (sb_id << 24) |
4549                        (TX_TSS_CID << 7));
4550                cp->tx_cons_ptr = &sblk->status_tx_quick_consumer_index;
4551        }
4552        cp->tx_cons = *cp->tx_cons_ptr;
4553
4554        cid_addr = GET_CID_ADDR(tx_cid);
4555        if (BNX2_CHIP(cp) == BNX2_CHIP_5709) {
4556                u32 cid_addr2 = GET_CID_ADDR(tx_cid + 4) + 0x40;
4557
4558                for (i = 0; i < PHY_CTX_SIZE; i += 4)
4559                        cnic_ctx_wr(dev, cid_addr2, i, 0);
4560
4561                offset0 = BNX2_L2CTX_TYPE_XI;
4562                offset1 = BNX2_L2CTX_CMD_TYPE_XI;
4563                offset2 = BNX2_L2CTX_TBDR_BHADDR_HI_XI;
4564                offset3 = BNX2_L2CTX_TBDR_BHADDR_LO_XI;
4565        } else {
4566                cnic_init_context(dev, tx_cid);
4567                cnic_init_context(dev, tx_cid + 1);
4568
4569                offset0 = BNX2_L2CTX_TYPE;
4570                offset1 = BNX2_L2CTX_CMD_TYPE;
4571                offset2 = BNX2_L2CTX_TBDR_BHADDR_HI;
4572                offset3 = BNX2_L2CTX_TBDR_BHADDR_LO;
4573        }
4574        val = BNX2_L2CTX_TYPE_TYPE_L2 | BNX2_L2CTX_TYPE_SIZE_L2;
4575        cnic_ctx_wr(dev, cid_addr, offset0, val);
4576
4577        val = BNX2_L2CTX_CMD_TYPE_TYPE_L2 | (8 << 16);
4578        cnic_ctx_wr(dev, cid_addr, offset1, val);
4579
4580        txbd = udev->l2_ring;
4581
4582        buf_map = udev->l2_buf_map;
4583        for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i++, txbd++) {
4584                txbd->tx_bd_haddr_hi = (u64) buf_map >> 32;
4585                txbd->tx_bd_haddr_lo = (u64) buf_map & 0xffffffff;
4586        }
4587        val = (u64) ring_map >> 32;
4588        cnic_ctx_wr(dev, cid_addr, offset2, val);
4589        txbd->tx_bd_haddr_hi = val;
4590
4591        val = (u64) ring_map & 0xffffffff;
4592        cnic_ctx_wr(dev, cid_addr, offset3, val);
4593        txbd->tx_bd_haddr_lo = val;
4594}
4595
4596static void cnic_init_bnx2_rx_ring(struct cnic_dev *dev)
4597{
4598        struct cnic_local *cp = dev->cnic_priv;
4599        struct cnic_eth_dev *ethdev = cp->ethdev;
4600        struct cnic_uio_dev *udev = cp->udev;
4601        u32 cid_addr, sb_id, val, coal_reg, coal_val;
4602        int i;
4603        struct bnx2_rx_bd *rxbd;
4604        struct status_block *s_blk = cp->status_blk.gen;
4605        dma_addr_t ring_map = udev->l2_ring_map;
4606
4607        sb_id = cp->status_blk_num;
4608        cnic_init_context(dev, 2);
4609        cp->rx_cons_ptr = &s_blk->status_rx_quick_consumer_index2;
4610        coal_reg = BNX2_HC_COMMAND;
4611        coal_val = CNIC_RD(dev, coal_reg);
4612        if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) {
4613                struct status_block_msix *sblk = cp->status_blk.bnx2;
4614
4615                cp->rx_cons_ptr = &sblk->status_rx_quick_consumer_index;
4616                coal_reg = BNX2_HC_COALESCE_NOW;
4617                coal_val = 1 << (11 + sb_id);
4618        }
4619        i = 0;
4620        while (!(*cp->rx_cons_ptr != 0) && i < 10) {
4621                CNIC_WR(dev, coal_reg, coal_val);
4622                udelay(10);
4623                i++;
4624                barrier();
4625        }
4626        cp->rx_cons = *cp->rx_cons_ptr;
4627
4628        cid_addr = GET_CID_ADDR(2);
4629        val = BNX2_L2CTX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE |
4630              BNX2_L2CTX_CTX_TYPE_SIZE_L2 | (0x02 << 8);
4631        cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_CTX_TYPE, val);
4632
4633        if (sb_id == 0)
4634                val = 2 << BNX2_L2CTX_L2_STATUSB_NUM_SHIFT;
4635        else
4636                val = BNX2_L2CTX_L2_STATUSB_NUM(sb_id);
4637        cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_HOST_BDIDX, val);
4638
4639        rxbd = udev->l2_ring + CNIC_PAGE_SIZE;
4640        for (i = 0; i < BNX2_MAX_RX_DESC_CNT; i++, rxbd++) {
4641                dma_addr_t buf_map;
4642                int n = (i % cp->l2_rx_ring_size) + 1;
4643
4644                buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size);
4645                rxbd->rx_bd_len = cp->l2_single_buf_size;
4646                rxbd->rx_bd_flags = RX_BD_FLAGS_START | RX_BD_FLAGS_END;
4647                rxbd->rx_bd_haddr_hi = (u64) buf_map >> 32;
4648                rxbd->rx_bd_haddr_lo = (u64) buf_map & 0xffffffff;
4649        }
4650        val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32;
4651        cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_HI, val);
4652        rxbd->rx_bd_haddr_hi = val;
4653
4654        val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff;
4655        cnic_ctx_wr(dev, cid_addr, BNX2_L2CTX_NX_BDHADDR_LO, val);
4656        rxbd->rx_bd_haddr_lo = val;
4657
4658        val = cnic_reg_rd_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD);
4659        cnic_reg_wr_ind(dev, BNX2_RXP_SCRATCH_RXP_FLOOD, val | (1 << 2));
4660}
4661
4662static void cnic_shutdown_bnx2_rx_ring(struct cnic_dev *dev)
4663{
4664        struct kwqe *wqes[1], l2kwqe;
4665
4666        memset(&l2kwqe, 0, sizeof(l2kwqe));
4667        wqes[0] = &l2kwqe;
4668        l2kwqe.kwqe_op_flag = (L2_LAYER_CODE << KWQE_LAYER_SHIFT) |
4669                              (L2_KWQE_OPCODE_VALUE_FLUSH <<
4670                               KWQE_OPCODE_SHIFT) | 2;
4671        dev->submit_kwqes(dev, wqes, 1);
4672}
4673
4674static void cnic_set_bnx2_mac(struct cnic_dev *dev)
4675{
4676        struct cnic_local *cp = dev->cnic_priv;
4677        u32 val;
4678
4679        val = cp->func << 2;
4680
4681        cp->shmem_base = cnic_reg_rd_ind(dev, BNX2_SHM_HDR_ADDR_0 + val);
4682
4683        val = cnic_reg_rd_ind(dev, cp->shmem_base +
4684                              BNX2_PORT_HW_CFG_ISCSI_MAC_UPPER);
4685        dev->mac_addr[0] = (u8) (val >> 8);
4686        dev->mac_addr[1] = (u8) val;
4687
4688        CNIC_WR(dev, BNX2_EMAC_MAC_MATCH4, val);
4689
4690        val = cnic_reg_rd_ind(dev, cp->shmem_base +
4691                              BNX2_PORT_HW_CFG_ISCSI_MAC_LOWER);
4692        dev->mac_addr[2] = (u8) (val >> 24);
4693        dev->mac_addr[3] = (u8) (val >> 16);
4694        dev->mac_addr[4] = (u8) (val >> 8);
4695        dev->mac_addr[5] = (u8) val;
4696
4697        CNIC_WR(dev, BNX2_EMAC_MAC_MATCH5, val);
4698
4699        val = 4 | BNX2_RPM_SORT_USER2_BC_EN;
4700        if (BNX2_CHIP(cp) != BNX2_CHIP_5709)
4701                val |= BNX2_RPM_SORT_USER2_PROM_VLAN;
4702
4703        CNIC_WR(dev, BNX2_RPM_SORT_USER2, 0x0);
4704        CNIC_WR(dev, BNX2_RPM_SORT_USER2, val);
4705        CNIC_WR(dev, BNX2_RPM_SORT_USER2, val | BNX2_RPM_SORT_USER2_ENA);
4706}
4707
4708static int cnic_start_bnx2_hw(struct cnic_dev *dev)
4709{
4710        struct cnic_local *cp = dev->cnic_priv;
4711        struct cnic_eth_dev *ethdev = cp->ethdev;
4712        struct status_block *sblk = cp->status_blk.gen;
4713        u32 val, kcq_cid_addr, kwq_cid_addr;
4714        int err;
4715
4716        cnic_set_bnx2_mac(dev);
4717
4718        val = CNIC_RD(dev, BNX2_MQ_CONFIG);
4719        val &= ~BNX2_MQ_CONFIG_KNL_BYP_BLK_SIZE;
4720        if (CNIC_PAGE_BITS > 12)
4721                val |= (12 - 8)  << 4;
4722        else
4723                val |= (CNIC_PAGE_BITS - 8)  << 4;
4724
4725        CNIC_WR(dev, BNX2_MQ_CONFIG, val);
4726
4727        CNIC_WR(dev, BNX2_HC_COMP_PROD_TRIP, (2 << 16) | 8);
4728        CNIC_WR(dev, BNX2_HC_COM_TICKS, (64 << 16) | 220);
4729        CNIC_WR(dev, BNX2_HC_CMD_TICKS, (64 << 16) | 220);
4730
4731        err = cnic_setup_5709_context(dev, 1);
4732        if (err)
4733                return err;
4734
4735        cnic_init_context(dev, KWQ_CID);
4736        cnic_init_context(dev, KCQ_CID);
4737
4738        kwq_cid_addr = GET_CID_ADDR(KWQ_CID);
4739        cp->kwq_io_addr = MB_GET_CID_ADDR(KWQ_CID) + L5_KRNLQ_HOST_QIDX;
4740
4741        cp->max_kwq_idx = MAX_KWQ_IDX;
4742        cp->kwq_prod_idx = 0;
4743        cp->kwq_con_idx = 0;
4744        set_bit(CNIC_LCL_FL_KWQ_INIT, &cp->cnic_local_flags);
4745
4746        if (BNX2_CHIP(cp) == BNX2_CHIP_5706 || BNX2_CHIP(cp) == BNX2_CHIP_5708)
4747                cp->kwq_con_idx_ptr = &sblk->status_rx_quick_consumer_index15;
4748        else
4749                cp->kwq_con_idx_ptr = &sblk->status_cmd_consumer_index;
4750
4751        /* Initialize the kernel work queue context. */
4752        val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE |
4753              (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ;
4754        cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_TYPE, val);
4755
4756        val = (CNIC_PAGE_SIZE / sizeof(struct kwqe) - 1) << 16;
4757        cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val);
4758
4759        val = ((CNIC_PAGE_SIZE / sizeof(struct kwqe)) << 16) | KWQ_PAGE_CNT;
4760        cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val);
4761
4762        val = (u32) ((u64) cp->kwq_info.pgtbl_map >> 32);
4763        cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val);
4764
4765        val = (u32) cp->kwq_info.pgtbl_map;
4766        cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val);
4767
4768        kcq_cid_addr = GET_CID_ADDR(KCQ_CID);
4769        cp->kcq1.io_addr = MB_GET_CID_ADDR(KCQ_CID) + L5_KRNLQ_HOST_QIDX;
4770
4771        cp->kcq1.sw_prod_idx = 0;
4772        cp->kcq1.hw_prod_idx_ptr =
4773                &sblk->status_completion_producer_index;
4774
4775        cp->kcq1.status_idx_ptr = &sblk->status_idx;
4776
4777        /* Initialize the kernel complete queue context. */
4778        val = KRNLQ_TYPE_TYPE_KRNLQ | KRNLQ_SIZE_TYPE_SIZE |
4779              (CNIC_PAGE_BITS - 8) | KRNLQ_FLAGS_QE_SELF_SEQ;
4780        cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_TYPE, val);
4781
4782        val = (CNIC_PAGE_SIZE / sizeof(struct kcqe) - 1) << 16;
4783        cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_QE_SELF_SEQ_MAX, val);
4784
4785        val = ((CNIC_PAGE_SIZE / sizeof(struct kcqe)) << 16) | KCQ_PAGE_CNT;
4786        cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_NPAGES, val);
4787
4788        val = (u32) ((u64) cp->kcq1.dma.pgtbl_map >> 32);
4789        cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_HI, val);
4790
4791        val = (u32) cp->kcq1.dma.pgtbl_map;
4792        cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_PGTBL_HADDR_LO, val);
4793
4794        cp->int_num = 0;
4795        if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX) {
4796                struct status_block_msix *msblk = cp->status_blk.bnx2;
4797                u32 sb_id = cp->status_blk_num;
4798                u32 sb = BNX2_L2CTX_L5_STATUSB_NUM(sb_id);
4799
4800                cp->kcq1.hw_prod_idx_ptr =
4801                        &msblk->status_completion_producer_index;
4802                cp->kcq1.status_idx_ptr = &msblk->status_idx;
4803                cp->kwq_con_idx_ptr = &msblk->status_cmd_consumer_index;
4804                cp->int_num = sb_id << BNX2_PCICFG_INT_ACK_CMD_INT_NUM_SHIFT;
4805                cnic_ctx_wr(dev, kwq_cid_addr, L5_KRNLQ_HOST_QIDX, sb);
4806                cnic_ctx_wr(dev, kcq_cid_addr, L5_KRNLQ_HOST_QIDX, sb);
4807        }
4808
4809        /* Enable Commnad Scheduler notification when we write to the
4810         * host producer index of the kernel contexts. */
4811        CNIC_WR(dev, BNX2_MQ_KNL_CMD_MASK1, 2);
4812
4813        /* Enable Command Scheduler notification when we write to either
4814         * the Send Queue or Receive Queue producer indexes of the kernel
4815         * bypass contexts. */
4816        CNIC_WR(dev, BNX2_MQ_KNL_BYP_CMD_MASK1, 7);
4817        CNIC_WR(dev, BNX2_MQ_KNL_BYP_WRITE_MASK1, 7);
4818
4819        /* Notify COM when the driver post an application buffer. */
4820        CNIC_WR(dev, BNX2_MQ_KNL_RX_V2P_MASK2, 0x2000);
4821
4822        /* Set the CP and COM doorbells.  These two processors polls the
4823         * doorbell for a non zero value before running.  This must be done
4824         * after setting up the kernel queue contexts. */
4825        cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 1);
4826        cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 1);
4827
4828        cnic_init_bnx2_tx_ring(dev);
4829        cnic_init_bnx2_rx_ring(dev);
4830
4831        err = cnic_init_bnx2_irq(dev);
4832        if (err) {
4833                netdev_err(dev->netdev, "cnic_init_irq failed\n");
4834                cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0);
4835                cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0);
4836                return err;
4837        }
4838
4839        ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ;
4840
4841        return 0;
4842}
4843
4844static void cnic_setup_bnx2x_context(struct cnic_dev *dev)
4845{
4846        struct cnic_local *cp = dev->cnic_priv;
4847        struct cnic_eth_dev *ethdev = cp->ethdev;
4848        u32 start_offset = ethdev->ctx_tbl_offset;
4849        int i;
4850
4851        for (i = 0; i < cp->ctx_blks; i++) {
4852                struct cnic_ctx *ctx = &cp->ctx_arr[i];
4853                dma_addr_t map = ctx->mapping;
4854
4855                if (cp->ctx_align) {
4856                        unsigned long mask = cp->ctx_align - 1;
4857
4858                        map = (map + mask) & ~mask;
4859                }
4860
4861                cnic_ctx_tbl_wr(dev, start_offset + i, map);
4862        }
4863}
4864
4865static int cnic_init_bnx2x_irq(struct cnic_dev *dev)
4866{
4867        struct cnic_local *cp = dev->cnic_priv;
4868        struct cnic_eth_dev *ethdev = cp->ethdev;
4869        int err = 0;
4870
4871        tasklet_init(&cp->cnic_irq_task, cnic_service_bnx2x_bh,
4872                     (unsigned long) dev);
4873        if (ethdev->drv_state & CNIC_DRV_STATE_USING_MSIX)
4874                err = cnic_request_irq(dev);
4875
4876        return err;
4877}
4878
4879static inline void cnic_storm_memset_hc_disable(struct cnic_dev *dev,
4880                                                u16 sb_id, u8 sb_index,
4881                                                u8 disable)
4882{
4883        struct bnx2x *bp = netdev_priv(dev->netdev);
4884
4885        u32 addr = BAR_CSTRORM_INTMEM +
4886                        CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) +
4887                        offsetof(struct hc_status_block_data_e1x, index_data) +
4888                        sizeof(struct hc_index_data)*sb_index +
4889                        offsetof(struct hc_index_data, flags);
4890        u16 flags = CNIC_RD16(dev, addr);
4891        /* clear and set */
4892        flags &= ~HC_INDEX_DATA_HC_ENABLED;
4893        flags |= (((~disable) << HC_INDEX_DATA_HC_ENABLED_SHIFT) &
4894                  HC_INDEX_DATA_HC_ENABLED);
4895        CNIC_WR16(dev, addr, flags);
4896}
4897
4898static void cnic_enable_bnx2x_int(struct cnic_dev *dev)
4899{
4900        struct cnic_local *cp = dev->cnic_priv;
4901        struct bnx2x *bp = netdev_priv(dev->netdev);
4902        u8 sb_id = cp->status_blk_num;
4903
4904        CNIC_WR8(dev, BAR_CSTRORM_INTMEM +
4905                        CSTORM_STATUS_BLOCK_DATA_OFFSET(sb_id) +
4906                        offsetof(struct hc_status_block_data_e1x, index_data) +
4907                        sizeof(struct hc_index_data)*HC_INDEX_ISCSI_EQ_CONS +
4908                        offsetof(struct hc_index_data, timeout), 64 / 4);
4909        cnic_storm_memset_hc_disable(dev, sb_id, HC_INDEX_ISCSI_EQ_CONS, 0);
4910}
4911
4912static void cnic_disable_bnx2x_int_sync(struct cnic_dev *dev)
4913{
4914}
4915
4916static void cnic_init_bnx2x_tx_ring(struct cnic_dev *dev,
4917                                    struct client_init_ramrod_data *data)
4918{
4919        struct cnic_local *cp = dev->cnic_priv;
4920        struct bnx2x *bp = netdev_priv(dev->netdev);
4921        struct cnic_uio_dev *udev = cp->udev;
4922        union eth_tx_bd_types *txbd = (union eth_tx_bd_types *) udev->l2_ring;
4923        dma_addr_t buf_map, ring_map = udev->l2_ring_map;
4924        struct host_sp_status_block *sb = cp->bnx2x_def_status_blk;
4925        int i;
4926        u32 cli = cp->ethdev->iscsi_l2_client_id;
4927        u32 val;
4928
4929        memset(txbd, 0, CNIC_PAGE_SIZE);
4930
4931        buf_map = udev->l2_buf_map;
4932        for (i = 0; i < BNX2_MAX_TX_DESC_CNT; i += 3, txbd += 3) {
4933                struct eth_tx_start_bd *start_bd = &txbd->start_bd;
4934                struct eth_tx_parse_bd_e1x *pbd_e1x =
4935                        &((txbd + 1)->parse_bd_e1x);
4936                struct eth_tx_parse_bd_e2 *pbd_e2 = &((txbd + 1)->parse_bd_e2);
4937                struct eth_tx_bd *reg_bd = &((txbd + 2)->reg_bd);
4938
4939                start_bd->addr_hi = cpu_to_le32((u64) buf_map >> 32);
4940                start_bd->addr_lo = cpu_to_le32(buf_map & 0xffffffff);
4941                reg_bd->addr_hi = start_bd->addr_hi;
4942                reg_bd->addr_lo = start_bd->addr_lo + 0x10;
4943                start_bd->nbytes = cpu_to_le16(0x10);
4944                start_bd->nbd = cpu_to_le16(3);
4945                start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
4946                start_bd->general_data &= ~ETH_TX_START_BD_PARSE_NBDS;
4947                start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
4948
4949                if (BNX2X_CHIP_IS_E2_PLUS(bp))
4950                        pbd_e2->parsing_data = (UNICAST_ADDRESS <<
4951                                ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE_SHIFT);
4952                else
4953                        pbd_e1x->global_data = (UNICAST_ADDRESS <<
4954                                ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE_SHIFT);
4955        }
4956
4957        val = (u64) ring_map >> 32;
4958        txbd->next_bd.addr_hi = cpu_to_le32(val);
4959
4960        data->tx.tx_bd_page_base.hi = cpu_to_le32(val);
4961
4962        val = (u64) ring_map & 0xffffffff;
4963        txbd->next_bd.addr_lo = cpu_to_le32(val);
4964
4965        data->tx.tx_bd_page_base.lo = cpu_to_le32(val);
4966
4967        /* Other ramrod params */
4968        data->tx.tx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_CQ_CONS;
4969        data->tx.tx_status_block_id = BNX2X_DEF_SB_ID;
4970
4971        /* reset xstorm per client statistics */
4972        if (cli < MAX_STAT_COUNTER_ID) {
4973                data->general.statistics_zero_flg = 1;
4974                data->general.statistics_en_flg = 1;
4975                data->general.statistics_counter_id = cli;
4976        }
4977
4978        cp->tx_cons_ptr =
4979                &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_CQ_CONS];
4980}
4981
4982static void cnic_init_bnx2x_rx_ring(struct cnic_dev *dev,
4983                                    struct client_init_ramrod_data *data)
4984{
4985        struct cnic_local *cp = dev->cnic_priv;
4986        struct bnx2x *bp = netdev_priv(dev->netdev);
4987        struct cnic_uio_dev *udev = cp->udev;
4988        struct eth_rx_bd *rxbd = (struct eth_rx_bd *) (udev->l2_ring +
4989                                CNIC_PAGE_SIZE);
4990        struct eth_rx_cqe_next_page *rxcqe = (struct eth_rx_cqe_next_page *)
4991                                (udev->l2_ring + (2 * CNIC_PAGE_SIZE));
4992        struct host_sp_status_block *sb = cp->bnx2x_def_status_blk;
4993        int i;
4994        u32 cli = cp->ethdev->iscsi_l2_client_id;
4995        int cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli);
4996        u32 val;
4997        dma_addr_t ring_map = udev->l2_ring_map;
4998
4999        /* General data */
5000        data->general.client_id = cli;
5001        data->general.activate_flg = 1;
5002        data->general.sp_client_id = cli;
5003        data->general.mtu = cpu_to_le16(cp->l2_single_buf_size - 14);
5004        data->general.func_id = bp->pfid;
5005
5006        for (i = 0; i < BNX2X_MAX_RX_DESC_CNT; i++, rxbd++) {
5007                dma_addr_t buf_map;
5008                int n = (i % cp->l2_rx_ring_size) + 1;
5009
5010                buf_map = udev->l2_buf_map + (n * cp->l2_single_buf_size);
5011                rxbd->addr_hi = cpu_to_le32((u64) buf_map >> 32);
5012                rxbd->addr_lo = cpu_to_le32(buf_map & 0xffffffff);
5013        }
5014
5015        val = (u64) (ring_map + CNIC_PAGE_SIZE) >> 32;
5016        rxbd->addr_hi = cpu_to_le32(val);
5017        data->rx.bd_page_base.hi = cpu_to_le32(val);
5018
5019        val = (u64) (ring_map + CNIC_PAGE_SIZE) & 0xffffffff;
5020        rxbd->addr_lo = cpu_to_le32(val);
5021        data->rx.bd_page_base.lo = cpu_to_le32(val);
5022
5023        rxcqe += BNX2X_MAX_RCQ_DESC_CNT;
5024        val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) >> 32;
5025        rxcqe->addr_hi = cpu_to_le32(val);
5026        data->rx.cqe_page_base.hi = cpu_to_le32(val);
5027
5028        val = (u64) (ring_map + (2 * CNIC_PAGE_SIZE)) & 0xffffffff;
5029        rxcqe->addr_lo = cpu_to_le32(val);
5030        data->rx.cqe_page_base.lo = cpu_to_le32(val);
5031
5032        /* Other ramrod params */
5033        data->rx.client_qzone_id = cl_qzone_id;
5034        data->rx.rx_sb_index_number = HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS;
5035        data->rx.status_block_id = BNX2X_DEF_SB_ID;
5036
5037        data->rx.cache_line_alignment_log_size = L1_CACHE_SHIFT;
5038
5039        data->rx.max_bytes_on_bd = cpu_to_le16(cp->l2_single_buf_size);
5040        data->rx.outer_vlan_removal_enable_flg = 1;
5041        data->rx.silent_vlan_removal_flg = 1;
5042        data->rx.silent_vlan_value = 0;
5043        data->rx.silent_vlan_mask = 0xffff;
5044
5045        cp->rx_cons_ptr =
5046                &sb->sp_sb.index_values[HC_SP_INDEX_ETH_ISCSI_RX_CQ_CONS];
5047        cp->rx_cons = *cp->rx_cons_ptr;
5048}
5049
5050static void cnic_init_bnx2x_kcq(struct cnic_dev *dev)
5051{
5052        struct cnic_local *cp = dev->cnic_priv;
5053        struct bnx2x *bp = netdev_priv(dev->netdev);
5054        u32 pfid = bp->pfid;
5055
5056        cp->kcq1.io_addr = BAR_CSTRORM_INTMEM +
5057                           CSTORM_ISCSI_EQ_PROD_OFFSET(pfid, 0);
5058        cp->kcq1.sw_prod_idx = 0;
5059
5060        if (BNX2X_CHIP_IS_E2_PLUS(bp)) {
5061                struct host_hc_status_block_e2 *sb = cp->status_blk.gen;
5062
5063                cp->kcq1.hw_prod_idx_ptr =
5064                        &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS];
5065                cp->kcq1.status_idx_ptr =
5066                        &sb->sb.running_index[SM_RX_ID];
5067        } else {
5068                struct host_hc_status_block_e1x *sb = cp->status_blk.gen;
5069
5070                cp->kcq1.hw_prod_idx_ptr =
5071                        &sb->sb.index_values[HC_INDEX_ISCSI_EQ_CONS];
5072                cp->kcq1.status_idx_ptr =
5073                        &sb->sb.running_index[SM_RX_ID];
5074        }
5075
5076        if (BNX2X_CHIP_IS_E2_PLUS(bp)) {
5077                struct host_hc_status_block_e2 *sb = cp->status_blk.gen;
5078
5079                cp->kcq2.io_addr = BAR_USTRORM_INTMEM +
5080                                        USTORM_FCOE_EQ_PROD_OFFSET(pfid);
5081                cp->kcq2.sw_prod_idx = 0;
5082                cp->kcq2.hw_prod_idx_ptr =
5083                        &sb->sb.index_values[HC_INDEX_FCOE_EQ_CONS];
5084                cp->kcq2.status_idx_ptr =
5085                        &sb->sb.running_index[SM_RX_ID];
5086        }
5087}
5088
5089static int cnic_start_bnx2x_hw(struct cnic_dev *dev)
5090{
5091        struct cnic_local *cp = dev->cnic_priv;
5092        struct bnx2x *bp = netdev_priv(dev->netdev);
5093        struct cnic_eth_dev *ethdev = cp->ethdev;
5094        int func, ret;
5095        u32 pfid;
5096
5097        dev->stats_addr = ethdev->addr_drv_info_to_mcp;
5098        cp->func = bp->pf_num;
5099
5100        func = CNIC_FUNC(cp);
5101        pfid = bp->pfid;
5102
5103        ret = cnic_init_id_tbl(&cp->cid_tbl, MAX_ISCSI_TBL_SZ,
5104                               cp->iscsi_start_cid, 0);
5105
5106        if (ret)
5107                return -ENOMEM;
5108
5109        if (BNX2X_CHIP_IS_E2_PLUS(bp)) {
5110                ret = cnic_init_id_tbl(&cp->fcoe_cid_tbl, dev->max_fcoe_conn,
5111                                        cp->fcoe_start_cid, 0);
5112
5113                if (ret)
5114                        return -ENOMEM;
5115        }
5116
5117        cp->bnx2x_igu_sb_id = ethdev->irq_arr[0].status_blk_num2;
5118
5119        cnic_init_bnx2x_kcq(dev);
5120
5121        /* Only 1 EQ */
5122        CNIC_WR16(dev, cp->kcq1.io_addr, MAX_KCQ_IDX);
5123        CNIC_WR(dev, BAR_CSTRORM_INTMEM +
5124                CSTORM_ISCSI_EQ_CONS_OFFSET(pfid, 0), 0);
5125        CNIC_WR(dev, BAR_CSTRORM_INTMEM +
5126                CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0),
5127                cp->kcq1.dma.pg_map_arr[1] & 0xffffffff);
5128        CNIC_WR(dev, BAR_CSTRORM_INTMEM +
5129                CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_OFFSET(pfid, 0) + 4,
5130                (u64) cp->kcq1.dma.pg_map_arr[1] >> 32);
5131        CNIC_WR(dev, BAR_CSTRORM_INTMEM +
5132                CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0),
5133                cp->kcq1.dma.pg_map_arr[0] & 0xffffffff);
5134        CNIC_WR(dev, BAR_CSTRORM_INTMEM +
5135                CSTORM_ISCSI_EQ_NEXT_EQE_ADDR_OFFSET(pfid, 0) + 4,
5136                (u64) cp->kcq1.dma.pg_map_arr[0] >> 32);
5137        CNIC_WR8(dev, BAR_CSTRORM_INTMEM +
5138                CSTORM_ISCSI_EQ_NEXT_PAGE_ADDR_VALID_OFFSET(pfid, 0), 1);
5139        CNIC_WR16(dev, BAR_CSTRORM_INTMEM +
5140                CSTORM_ISCSI_EQ_SB_NUM_OFFSET(pfid, 0), cp->status_blk_num);
5141        CNIC_WR8(dev, BAR_CSTRORM_INTMEM +
5142                CSTORM_ISCSI_EQ_SB_INDEX_OFFSET(pfid, 0),
5143                HC_INDEX_ISCSI_EQ_CONS);
5144
5145        CNIC_WR(dev, BAR_USTRORM_INTMEM +
5146                USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid),
5147                cp->gbl_buf_info.pg_map_arr[0] & 0xffffffff);
5148        CNIC_WR(dev, BAR_USTRORM_INTMEM +
5149                USTORM_ISCSI_GLOBAL_BUF_PHYS_ADDR_OFFSET(pfid) + 4,
5150                (u64) cp->gbl_buf_info.pg_map_arr[0] >> 32);
5151
5152        CNIC_WR(dev, BAR_TSTRORM_INTMEM +
5153                TSTORM_ISCSI_TCP_LOCAL_ADV_WND_OFFSET(pfid), DEF_RCV_BUF);
5154
5155        cnic_setup_bnx2x_context(dev);
5156
5157        ret = cnic_init_bnx2x_irq(dev);
5158        if (ret)
5159                return ret;
5160
5161        ethdev->drv_state |= CNIC_DRV_STATE_HANDLES_IRQ;
5162        return 0;
5163}
5164
5165static void cnic_init_rings(struct cnic_dev *dev)
5166{
5167        struct cnic_local *cp = dev->cnic_priv;
5168        struct bnx2x *bp = netdev_priv(dev->netdev);
5169        struct cnic_uio_dev *udev = cp->udev;
5170
5171        if (test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags))
5172                return;
5173
5174        if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) {
5175                cnic_init_bnx2_tx_ring(dev);
5176                cnic_init_bnx2_rx_ring(dev);
5177                set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags);
5178        } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) {
5179                u32 cli = cp->ethdev->iscsi_l2_client_id;
5180                u32 cid = cp->ethdev->iscsi_l2_cid;
5181                u32 cl_qzone_id;
5182                struct client_init_ramrod_data *data;
5183                union l5cm_specific_data l5_data;
5184                struct ustorm_eth_rx_producers rx_prods = {0};
5185                u32 off, i, *cid_ptr;
5186
5187                rx_prods.bd_prod = 0;
5188                rx_prods.cqe_prod = BNX2X_MAX_RCQ_DESC_CNT;
5189                barrier();
5190
5191                cl_qzone_id = BNX2X_CL_QZONE_ID(bp, cli);
5192
5193                off = BAR_USTRORM_INTMEM +
5194                        (BNX2X_CHIP_IS_E2_PLUS(bp) ?
5195                         USTORM_RX_PRODS_E2_OFFSET(cl_qzone_id) :
5196                         USTORM_RX_PRODS_E1X_OFFSET(BP_PORT(bp), cli));
5197
5198                for (i = 0; i < sizeof(struct ustorm_eth_rx_producers) / 4; i++)
5199                        CNIC_WR(dev, off + i * 4, ((u32 *) &rx_prods)[i]);
5200
5201                set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags);
5202
5203                data = udev->l2_buf;
5204                cid_ptr = udev->l2_buf + 12;
5205
5206                memset(data, 0, sizeof(*data));
5207
5208                cnic_init_bnx2x_tx_ring(dev, data);
5209                cnic_init_bnx2x_rx_ring(dev, data);
5210
5211                l5_data.phy_address.lo = udev->l2_buf_map & 0xffffffff;
5212                l5_data.phy_address.hi = (u64) udev->l2_buf_map >> 32;
5213
5214                set_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags);
5215
5216                cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_CLIENT_SETUP,
5217                        cid, ETH_CONNECTION_TYPE, &l5_data);
5218
5219                i = 0;
5220                while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) &&
5221                       ++i < 10)
5222                        msleep(1);
5223
5224                if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags))
5225                        netdev_err(dev->netdev,
5226                                "iSCSI CLIENT_SETUP did not complete\n");
5227                cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1);
5228                cnic_ring_ctl(dev, cid, cli, 1);
5229                *cid_ptr = cid >> 4;
5230                *(cid_ptr + 1) = cid * bp->db_size;
5231                *(cid_ptr + 2) = UIO_USE_TX_DOORBELL;
5232        }
5233}
5234
5235static void cnic_shutdown_rings(struct cnic_dev *dev)
5236{
5237        struct cnic_local *cp = dev->cnic_priv;
5238        struct cnic_uio_dev *udev = cp->udev;
5239        void *rx_ring;
5240
5241        if (!test_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags))
5242                return;
5243
5244        if (test_bit(CNIC_F_BNX2_CLASS, &dev->flags)) {
5245                cnic_shutdown_bnx2_rx_ring(dev);
5246        } else if (test_bit(CNIC_F_BNX2X_CLASS, &dev->flags)) {
5247                u32 cli = cp->ethdev->iscsi_l2_client_id;
5248                u32 cid = cp->ethdev->iscsi_l2_cid;
5249                union l5cm_specific_data l5_data;
5250                int i;
5251
5252                cnic_ring_ctl(dev, cid, cli, 0);
5253
5254                set_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags);
5255
5256                l5_data.phy_address.lo = cli;
5257                l5_data.phy_address.hi = 0;
5258                cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_ETH_HALT,
5259                        cid, ETH_CONNECTION_TYPE, &l5_data);
5260                i = 0;
5261                while (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags) &&
5262                       ++i < 10)
5263                        msleep(1);
5264
5265                if (test_bit(CNIC_LCL_FL_L2_WAIT, &cp->cnic_local_flags))
5266                        netdev_err(dev->netdev,
5267                                "iSCSI CLIENT_HALT did not complete\n");
5268                cnic_spq_completion(dev, DRV_CTL_RET_L2_SPQ_CREDIT_CMD, 1);
5269
5270                memset(&l5_data, 0, sizeof(l5_data));
5271                cnic_submit_kwqe_16(dev, RAMROD_CMD_ID_COMMON_CFC_DEL,
5272                        cid, NONE_CONNECTION_TYPE, &l5_data);
5273                msleep(10);
5274        }
5275        clear_bit(CNIC_LCL_FL_RINGS_INITED, &cp->cnic_local_flags);
5276        rx_ring = udev->l2_ring + CNIC_PAGE_SIZE;
5277        memset(rx_ring, 0, CNIC_PAGE_SIZE);
5278}
5279
5280static int cnic_register_netdev(struct cnic_dev *dev)
5281{
5282        struct cnic_local *cp = dev->cnic_priv;
5283        struct cnic_eth_dev *ethdev = cp->ethdev;
5284        int err;
5285
5286        if (!ethdev)
5287                return -ENODEV;
5288
5289        if (ethdev->drv_state & CNIC_DRV_STATE_REGD)
5290                return 0;
5291
5292        err = ethdev->drv_register_cnic(dev->netdev, cp->cnic_ops, dev);
5293        if (err)
5294                netdev_err(dev->netdev, "register_cnic failed\n");
5295
5296        /* Read iSCSI config again.  On some bnx2x device, iSCSI config
5297         * can change after firmware is downloaded.
5298         */
5299        dev->max_iscsi_conn = ethdev->max_iscsi_conn;
5300        if (ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI)
5301                dev->max_iscsi_conn = 0;
5302
5303        return err;
5304}
5305
5306static void cnic_unregister_netdev(struct cnic_dev *dev)
5307{
5308        struct cnic_local *cp = dev->cnic_priv;
5309        struct cnic_eth_dev *ethdev = cp->ethdev;
5310
5311        if (!ethdev)
5312                return;
5313
5314        ethdev->drv_unregister_cnic(dev->netdev);
5315}
5316
5317static int cnic_start_hw(struct cnic_dev *dev)
5318{
5319        struct cnic_local *cp = dev->cnic_priv;
5320        struct cnic_eth_dev *ethdev = cp->ethdev;
5321        int err;
5322
5323        if (test_bit(CNIC_F_CNIC_UP, &dev->flags))
5324                return -EALREADY;
5325
5326        dev->regview = ethdev->io_base;
5327        pci_dev_get(dev->pcidev);
5328        cp->func = PCI_FUNC(dev->pcidev->devfn);
5329        cp->status_blk.gen = ethdev->irq_arr[0].status_blk;
5330        cp->status_blk_num = ethdev->irq_arr[0].status_blk_num;
5331
5332        err = cp->alloc_resc(dev);
5333        if (err) {
5334                netdev_err(dev->netdev, "allocate resource failure\n");
5335                goto err1;
5336        }
5337
5338        err = cp->start_hw(dev);
5339        if (err)
5340                goto err1;
5341
5342        err = cnic_cm_open(dev);
5343        if (err)
5344                goto err1;
5345
5346        set_bit(CNIC_F_CNIC_UP, &dev->flags);
5347
5348        cp->enable_int(dev);
5349
5350        return 0;
5351
5352err1:
5353        if (ethdev->drv_state & CNIC_DRV_STATE_HANDLES_IRQ)
5354                cp->stop_hw(dev);
5355        else
5356                cp->free_resc(dev);
5357        pci_dev_put(dev->pcidev);
5358        return err;
5359}
5360
5361static void cnic_stop_bnx2_hw(struct cnic_dev *dev)
5362{
5363        cnic_disable_bnx2_int_sync(dev);
5364
5365        cnic_reg_wr_ind(dev, BNX2_CP_SCRATCH + 0x20, 0);
5366        cnic_reg_wr_ind(dev, BNX2_COM_SCRATCH + 0x20, 0);
5367
5368        cnic_init_context(dev, KWQ_CID);
5369        cnic_init_context(dev, KCQ_CID);
5370
5371        cnic_setup_5709_context(dev, 0);
5372        cnic_free_irq(dev);
5373
5374        cnic_free_resc(dev);
5375}
5376
5377
5378static void cnic_stop_bnx2x_hw(struct cnic_dev *dev)
5379{
5380        struct cnic_local *cp = dev->cnic_priv;
5381        struct bnx2x *bp = netdev_priv(dev->netdev);
5382        u32 hc_index = HC_INDEX_ISCSI_EQ_CONS;
5383        u32 sb_id = cp->status_blk_num;
5384        u32 idx_off, syn_off;
5385
5386        cnic_free_irq(dev);
5387
5388        if (BNX2X_CHIP_IS_E2_PLUS(bp)) {
5389                idx_off = offsetof(struct hc_status_block_e2, index_values) +
5390                          (hc_index * sizeof(u16));
5391
5392                syn_off = CSTORM_HC_SYNC_LINE_INDEX_E2_OFFSET(hc_index, sb_id);
5393        } else {
5394                idx_off = offsetof(struct hc_status_block_e1x, index_values) +
5395                          (hc_index * sizeof(u16));
5396
5397                syn_off = CSTORM_HC_SYNC_LINE_INDEX_E1X_OFFSET(hc_index, sb_id);
5398        }
5399        CNIC_WR16(dev, BAR_CSTRORM_INTMEM + syn_off, 0);
5400        CNIC_WR16(dev, BAR_CSTRORM_INTMEM + CSTORM_STATUS_BLOCK_OFFSET(sb_id) +
5401                  idx_off, 0);
5402
5403        *cp->kcq1.hw_prod_idx_ptr = 0;
5404        CNIC_WR(dev, BAR_CSTRORM_INTMEM +
5405                CSTORM_ISCSI_EQ_CONS_OFFSET(bp->pfid, 0), 0);
5406        CNIC_WR16(dev, cp->kcq1.io_addr, 0);
5407        cnic_free_resc(dev);
5408}
5409
5410static void cnic_stop_hw(struct cnic_dev *dev)
5411{
5412        if (test_bit(CNIC_F_CNIC_UP, &dev->flags)) {
5413                struct cnic_local *cp = dev->cnic_priv;
5414                int i = 0;
5415
5416                /* Need to wait for the ring shutdown event to complete
5417                 * before clearing the CNIC_UP flag.
5418                 */
5419                while (cp->udev && cp->udev->uio_dev != -1 && i < 15) {
5420                        msleep(100);
5421                        i++;
5422                }
5423                cnic_shutdown_rings(dev);
5424                cp->stop_cm(dev);
5425                cp->ethdev->drv_state &= ~CNIC_DRV_STATE_HANDLES_IRQ;
5426                clear_bit(CNIC_F_CNIC_UP, &dev->flags);
5427                RCU_INIT_POINTER(cp->ulp_ops[CNIC_ULP_L4], NULL);
5428                synchronize_rcu();
5429                cnic_cm_shutdown(dev);
5430                cp->stop_hw(dev);
5431                pci_dev_put(dev->pcidev);
5432        }
5433}
5434
5435static void cnic_free_dev(struct cnic_dev *dev)
5436{
5437        int i = 0;
5438
5439        while ((atomic_read(&dev->ref_count) != 0) && i < 10) {
5440                msleep(100);
5441                i++;
5442        }
5443        if (atomic_read(&dev->ref_count) != 0)
5444                netdev_err(dev->netdev, "Failed waiting for ref count to go to zero\n");
5445
5446        netdev_info(dev->netdev, "Removed CNIC device\n");
5447        dev_put(dev->netdev);
5448        kfree(dev);
5449}
5450
5451static int cnic_get_fc_npiv_tbl(struct cnic_dev *dev,
5452                                struct cnic_fc_npiv_tbl *npiv_tbl)
5453{
5454        struct cnic_local *cp = dev->cnic_priv;
5455        struct bnx2x *bp = netdev_priv(dev->netdev);
5456        int ret;
5457
5458        if (!test_bit(CNIC_F_CNIC_UP, &dev->flags))
5459                return -EAGAIN;     /* bnx2x is down */
5460
5461        if (!BNX2X_CHIP_IS_E2_PLUS(bp))
5462                return -EINVAL;
5463
5464        ret = cp->ethdev->drv_get_fc_npiv_tbl(dev->netdev, npiv_tbl);
5465        return ret;
5466}
5467
5468static struct cnic_dev *cnic_alloc_dev(struct net_device *dev,
5469                                       struct pci_dev *pdev)
5470{
5471        struct cnic_dev *cdev;
5472        struct cnic_local *cp;
5473        int alloc_size;
5474
5475        alloc_size = sizeof(struct cnic_dev) + sizeof(struct cnic_local);
5476
5477        cdev = kzalloc(alloc_size, GFP_KERNEL);
5478        if (cdev == NULL)
5479                return NULL;
5480
5481        cdev->netdev = dev;
5482        cdev->cnic_priv = (char *)cdev + sizeof(struct cnic_dev);
5483        cdev->register_device = cnic_register_device;
5484        cdev->unregister_device = cnic_unregister_device;
5485        cdev->iscsi_nl_msg_recv = cnic_iscsi_nl_msg_recv;
5486        cdev->get_fc_npiv_tbl = cnic_get_fc_npiv_tbl;
5487
5488        cp = cdev->cnic_priv;
5489        cp->dev = cdev;
5490        cp->l2_single_buf_size = 0x400;
5491        cp->l2_rx_ring_size = 3;
5492
5493        spin_lock_init(&cp->cnic_ulp_lock);
5494
5495        netdev_info(dev, "Added CNIC device\n");
5496
5497        return cdev;
5498}
5499
5500static struct cnic_dev *init_bnx2_cnic(struct net_device *dev)
5501{
5502        struct pci_dev *pdev;
5503        struct cnic_dev *cdev;
5504        struct cnic_local *cp;
5505        struct bnx2 *bp = netdev_priv(dev);
5506        struct cnic_eth_dev *ethdev = NULL;
5507
5508        if (bp->cnic_probe)
5509                ethdev = (bp->cnic_probe)(dev);
5510
5511        if (!ethdev)
5512                return NULL;
5513
5514        pdev = ethdev->pdev;
5515        if (!pdev)
5516                return NULL;
5517
5518        dev_hold(dev);
5519        pci_dev_get(pdev);
5520        if ((pdev->device == PCI_DEVICE_ID_NX2_5709 ||
5521             pdev->device == PCI_DEVICE_ID_NX2_5709S) &&
5522            (pdev->revision < 0x10)) {
5523                pci_dev_put(pdev);
5524                goto cnic_err;
5525        }
5526        pci_dev_put(pdev);
5527
5528        cdev = cnic_alloc_dev(dev, pdev);
5529        if (cdev == NULL)
5530                goto cnic_err;
5531
5532        set_bit(CNIC_F_BNX2_CLASS, &cdev->flags);
5533        cdev->submit_kwqes = cnic_submit_bnx2_kwqes;
5534
5535        cp = cdev->cnic_priv;
5536        cp->ethdev = ethdev;
5537        cdev->pcidev = pdev;
5538        cp->chip_id = ethdev->chip_id;
5539
5540        cdev->max_iscsi_conn = ethdev->max_iscsi_conn;
5541
5542        cp->cnic_ops = &cnic_bnx2_ops;
5543        cp->start_hw = cnic_start_bnx2_hw;
5544        cp->stop_hw = cnic_stop_bnx2_hw;
5545        cp->setup_pgtbl = cnic_setup_page_tbl;
5546        cp->alloc_resc = cnic_alloc_bnx2_resc;
5547        cp->free_resc = cnic_free_resc;
5548        cp->start_cm = cnic_cm_init_bnx2_hw;
5549        cp->stop_cm = cnic_cm_stop_bnx2_hw;
5550        cp->enable_int = cnic_enable_bnx2_int;
5551        cp->disable_int_sync = cnic_disable_bnx2_int_sync;
5552        cp->close_conn = cnic_close_bnx2_conn;
5553        return cdev;
5554
5555cnic_err:
5556        dev_put(dev);
5557        return NULL;
5558}
5559
5560static struct cnic_dev *init_bnx2x_cnic(struct net_device *dev)
5561{
5562        struct pci_dev *pdev;
5563        struct cnic_dev *cdev;
5564        struct cnic_local *cp;
5565        struct bnx2x *bp = netdev_priv(dev);
5566        struct cnic_eth_dev *ethdev = NULL;
5567
5568        if (bp->cnic_probe)
5569                ethdev = bp->cnic_probe(dev);
5570
5571        if (!ethdev)
5572                return NULL;
5573
5574        pdev = ethdev->pdev;
5575        if (!pdev)
5576                return NULL;
5577
5578        dev_hold(dev);
5579        cdev = cnic_alloc_dev(dev, pdev);
5580        if (cdev == NULL) {
5581                dev_put(dev);
5582                return NULL;
5583        }
5584
5585        set_bit(CNIC_F_BNX2X_CLASS, &cdev->flags);
5586        cdev->submit_kwqes = cnic_submit_bnx2x_kwqes;
5587
5588        cp = cdev->cnic_priv;
5589        cp->ethdev = ethdev;
5590        cdev->pcidev = pdev;
5591        cp->chip_id = ethdev->chip_id;
5592
5593        cdev->stats_addr = ethdev->addr_drv_info_to_mcp;
5594
5595        if (!(ethdev->drv_state & CNIC_DRV_STATE_NO_ISCSI))
5596                cdev->max_iscsi_conn = ethdev->max_iscsi_conn;
5597        if (CNIC_SUPPORTS_FCOE(bp)) {
5598                cdev->max_fcoe_conn = ethdev->max_fcoe_conn;
5599                cdev->max_fcoe_exchanges = ethdev->max_fcoe_exchanges;
5600        }
5601
5602        if (cdev->max_fcoe_conn > BNX2X_FCOE_NUM_CONNECTIONS)
5603                cdev->max_fcoe_conn = BNX2X_FCOE_NUM_CONNECTIONS;
5604
5605        memcpy(cdev->mac_addr, ethdev->iscsi_mac, ETH_ALEN);
5606
5607        cp->cnic_ops = &cnic_bnx2x_ops;
5608        cp->start_hw = cnic_start_bnx2x_hw;
5609        cp->stop_hw = cnic_stop_bnx2x_hw;
5610        cp->setup_pgtbl = cnic_setup_page_tbl_le;
5611        cp->alloc_resc = cnic_alloc_bnx2x_resc;
5612        cp->free_resc = cnic_free_resc;
5613        cp->start_cm = cnic_cm_init_bnx2x_hw;
5614        cp->stop_cm = cnic_cm_stop_bnx2x_hw;
5615        cp->enable_int = cnic_enable_bnx2x_int;
5616        cp->disable_int_sync = cnic_disable_bnx2x_int_sync;
5617        if (BNX2X_CHIP_IS_E2_PLUS(bp)) {
5618                cp->ack_int = cnic_ack_bnx2x_e2_msix;
5619                cp->arm_int = cnic_arm_bnx2x_e2_msix;
5620        } else {
5621                cp->ack_int = cnic_ack_bnx2x_msix;
5622                cp->arm_int = cnic_arm_bnx2x_msix;
5623        }
5624        cp->close_conn = cnic_close_bnx2x_conn;
5625        return cdev;
5626}
5627
5628static struct cnic_dev *is_cnic_dev(struct net_device *dev)
5629{
5630        struct ethtool_drvinfo drvinfo;
5631        struct cnic_dev *cdev = NULL;
5632
5633        if (dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) {
5634                memset(&drvinfo, 0, sizeof(drvinfo));
5635                dev->ethtool_ops->get_drvinfo(dev, &drvinfo);
5636
5637                if (!strcmp(drvinfo.driver, "bnx2"))
5638                        cdev = init_bnx2_cnic(dev);
5639                if (!strcmp(drvinfo.driver, "bnx2x"))
5640                        cdev = init_bnx2x_cnic(dev);
5641                if (cdev) {
5642                        write_lock(&cnic_dev_lock);
5643                        list_add(&cdev->list, &cnic_dev_list);
5644                        write_unlock(&cnic_dev_lock);
5645                }
5646        }
5647        return cdev;
5648}
5649
5650static void cnic_rcv_netevent(struct cnic_local *cp, unsigned long event,
5651                              u16 vlan_id)
5652{
5653        int if_type;
5654
5655        for (if_type = 0; if_type < MAX_CNIC_ULP_TYPE; if_type++) {
5656                struct cnic_ulp_ops *ulp_ops;
5657                void *ctx;
5658
5659                mutex_lock(&cnic_lock);
5660                ulp_ops = rcu_dereference_protected(cp->ulp_ops[if_type],
5661                                                lockdep_is_held(&cnic_lock));
5662                if (!ulp_ops || !ulp_ops->indicate_netevent) {
5663                        mutex_unlock(&cnic_lock);
5664                        continue;
5665                }
5666
5667                ctx = cp->ulp_handle[if_type];
5668
5669                set_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
5670                mutex_unlock(&cnic_lock);
5671
5672                ulp_ops->indicate_netevent(ctx, event, vlan_id);
5673
5674                clear_bit(ULP_F_CALL_PENDING, &cp->ulp_flags[if_type]);
5675        }
5676}
5677
5678/* netdev event handler */
5679static int cnic_netdev_event(struct notifier_block *this, unsigned long event,
5680                                                         void *ptr)
5681{
5682        struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
5683        struct cnic_dev *dev;
5684        int new_dev = 0;
5685
5686        dev = cnic_from_netdev(netdev);
5687
5688        if (!dev && event == NETDEV_REGISTER) {
5689                /* Check for the hot-plug device */
5690                dev = is_cnic_dev(netdev);
5691                if (dev) {
5692                        new_dev = 1;
5693                        cnic_hold(dev);
5694                }
5695        }
5696        if (dev) {
5697                struct cnic_local *cp = dev->cnic_priv;
5698
5699                if (new_dev)
5700                        cnic_ulp_init(dev);
5701                else if (event == NETDEV_UNREGISTER)
5702                        cnic_ulp_exit(dev);
5703
5704                if (event == NETDEV_UP) {
5705                        if (cnic_register_netdev(dev) != 0) {
5706                                cnic_put(dev);
5707                                goto done;
5708                        }
5709                        if (!cnic_start_hw(dev))
5710                                cnic_ulp_start(dev);
5711                }
5712
5713                cnic_rcv_netevent(cp, event, 0);
5714
5715                if (event == NETDEV_GOING_DOWN) {
5716                        cnic_ulp_stop(dev);
5717                        cnic_stop_hw(dev);
5718                        cnic_unregister_netdev(dev);
5719                } else if (event == NETDEV_UNREGISTER) {
5720                        write_lock(&cnic_dev_lock);
5721                        list_del_init(&dev->list);
5722                        write_unlock(&cnic_dev_lock);
5723
5724                        cnic_put(dev);
5725                        cnic_free_dev(dev);
5726                        goto done;
5727                }
5728                cnic_put(dev);
5729        } else {
5730                struct net_device *realdev;
5731                u16 vid;
5732
5733                vid = cnic_get_vlan(netdev, &realdev);
5734                if (realdev) {
5735                        dev = cnic_from_netdev(realdev);
5736                        if (dev) {
5737                                vid |= VLAN_TAG_PRESENT;
5738                                cnic_rcv_netevent(dev->cnic_priv, event, vid);
5739                                cnic_put(dev);
5740                        }
5741                }
5742        }
5743done:
5744        return NOTIFY_DONE;
5745}
5746
5747static struct notifier_block cnic_netdev_notifier = {
5748        .notifier_call = cnic_netdev_event
5749};
5750
5751static void cnic_release(void)
5752{
5753        struct cnic_uio_dev *udev;
5754
5755        while (!list_empty(&cnic_udev_list)) {
5756                udev = list_entry(cnic_udev_list.next, struct cnic_uio_dev,
5757                                  list);
5758                cnic_free_uio(udev);
5759        }
5760}
5761
5762static int __init cnic_init(void)
5763{
5764        int rc = 0;
5765
5766        pr_info("%s", version);
5767
5768        rc = register_netdevice_notifier(&cnic_netdev_notifier);
5769        if (rc) {
5770                cnic_release();
5771                return rc;
5772        }
5773
5774        cnic_wq = create_singlethread_workqueue("cnic_wq");
5775        if (!cnic_wq) {
5776                cnic_release();
5777                unregister_netdevice_notifier(&cnic_netdev_notifier);
5778                return -ENOMEM;
5779        }
5780
5781        return 0;
5782}
5783
5784static void __exit cnic_exit(void)
5785{
5786        unregister_netdevice_notifier(&cnic_netdev_notifier);
5787        cnic_release();
5788        destroy_workqueue(cnic_wq);
5789}
5790
5791module_init(cnic_init);
5792module_exit(cnic_exit);
5793