linux/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
<<
>>
Prefs
   1/*
   2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
   3 *
   4 * Copyright (C) 2003-2015 Chelsio Communications.  All rights reserved.
   5 *
   6 * This program is distributed in the hope that it will be useful, but WITHOUT
   7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   8 * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
   9 * release for licensing terms and conditions.
  10 *
  11 * Written by:  Dimitris Michailidis (dm@chelsio.com)
  12 *              Karen Xie (kxie@chelsio.com)
  13 */
  14
  15#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
  16
  17#include <linux/module.h>
  18#include <linux/moduleparam.h>
  19#include <scsi/scsi_host.h>
  20
  21#include "common.h"
  22#include "t3_cpl.h"
  23#include "t3cdev.h"
  24#include "cxgb3_defs.h"
  25#include "cxgb3_ctl_defs.h"
  26#include "cxgb3_offload.h"
  27#include "firmware_exports.h"
  28#include "cxgb3i.h"
  29
  30static unsigned int dbg_level;
  31#include "../libcxgbi.h"
  32
  33#define DRV_MODULE_NAME         "cxgb3i"
  34#define DRV_MODULE_DESC         "Chelsio T3 iSCSI Driver"
  35#define DRV_MODULE_VERSION      "2.0.1-ko"
  36#define DRV_MODULE_RELDATE      "Apr. 2015"
  37
  38static char version[] =
  39        DRV_MODULE_DESC " " DRV_MODULE_NAME
  40        " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  41
  42MODULE_AUTHOR("Chelsio Communications, Inc.");
  43MODULE_DESCRIPTION(DRV_MODULE_DESC);
  44MODULE_VERSION(DRV_MODULE_VERSION);
  45MODULE_LICENSE("GPL");
  46
  47module_param(dbg_level, uint, 0644);
  48MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");
  49
  50static int cxgb3i_rcv_win = 256 * 1024;
  51module_param(cxgb3i_rcv_win, int, 0644);
  52MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");
  53
  54static int cxgb3i_snd_win = 128 * 1024;
  55module_param(cxgb3i_snd_win, int, 0644);
  56MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
  57
  58static int cxgb3i_rx_credit_thres = 10 * 1024;
  59module_param(cxgb3i_rx_credit_thres, int, 0644);
  60MODULE_PARM_DESC(cxgb3i_rx_credit_thres,
  61                 "RX credits return threshold in bytes (default=10KB)");
  62
  63static unsigned int cxgb3i_max_connect = 8 * 1024;
  64module_param(cxgb3i_max_connect, uint, 0644);
  65MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");
  66
  67static unsigned int cxgb3i_sport_base = 20000;
  68module_param(cxgb3i_sport_base, uint, 0644);
  69MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");
  70
  71static void cxgb3i_dev_open(struct t3cdev *);
  72static void cxgb3i_dev_close(struct t3cdev *);
  73static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);
  74
  75static struct cxgb3_client t3_client = {
  76        .name = DRV_MODULE_NAME,
  77        .handlers = cxgb3i_cpl_handlers,
  78        .add = cxgb3i_dev_open,
  79        .remove = cxgb3i_dev_close,
  80        .event_handler = cxgb3i_dev_event_handler,
  81};
  82
  83static struct scsi_host_template cxgb3i_host_template = {
  84        .module         = THIS_MODULE,
  85        .name           = DRV_MODULE_NAME,
  86        .proc_name      = DRV_MODULE_NAME,
  87        .can_queue      = CXGB3I_SCSI_HOST_QDEPTH,
  88        .queuecommand   = iscsi_queuecommand,
  89        .change_queue_depth = scsi_change_queue_depth,
  90        .sg_tablesize   = SG_ALL,
  91        .max_sectors    = 0xFFFF,
  92        .cmd_per_lun    = ISCSI_DEF_CMD_PER_LUN,
  93        .eh_timed_out   = iscsi_eh_cmd_timed_out,
  94        .eh_abort_handler = iscsi_eh_abort,
  95        .eh_device_reset_handler = iscsi_eh_device_reset,
  96        .eh_target_reset_handler = iscsi_eh_recover_target,
  97        .target_alloc   = iscsi_target_alloc,
  98        .use_clustering = DISABLE_CLUSTERING,
  99        .this_id        = -1,
 100        .track_queue_depth = 1,
 101};
 102
 103static struct iscsi_transport cxgb3i_iscsi_transport = {
 104        .owner          = THIS_MODULE,
 105        .name           = DRV_MODULE_NAME,
 106        /* owner and name should be set already */
 107        .caps           = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
 108                                | CAP_DATADGST | CAP_DIGEST_OFFLOAD |
 109                                CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
 110        .attr_is_visible        = cxgbi_attr_is_visible,
 111        .get_host_param = cxgbi_get_host_param,
 112        .set_host_param = cxgbi_set_host_param,
 113        /* session management */
 114        .create_session = cxgbi_create_session,
 115        .destroy_session        = cxgbi_destroy_session,
 116        .get_session_param = iscsi_session_get_param,
 117        /* connection management */
 118        .create_conn    = cxgbi_create_conn,
 119        .bind_conn      = cxgbi_bind_conn,
 120        .destroy_conn   = iscsi_tcp_conn_teardown,
 121        .start_conn     = iscsi_conn_start,
 122        .stop_conn      = iscsi_conn_stop,
 123        .get_conn_param = iscsi_conn_get_param,
 124        .set_param      = cxgbi_set_conn_param,
 125        .get_stats      = cxgbi_get_conn_stats,
 126        /* pdu xmit req from user space */
 127        .send_pdu       = iscsi_conn_send_pdu,
 128        /* task */
 129        .init_task      = iscsi_tcp_task_init,
 130        .xmit_task      = iscsi_tcp_task_xmit,
 131        .cleanup_task   = cxgbi_cleanup_task,
 132        /* pdu */
 133        .alloc_pdu      = cxgbi_conn_alloc_pdu,
 134        .init_pdu       = cxgbi_conn_init_pdu,
 135        .xmit_pdu       = cxgbi_conn_xmit_pdu,
 136        .parse_pdu_itt  = cxgbi_parse_pdu_itt,
 137        /* TCP connect/disconnect */
 138        .get_ep_param   = cxgbi_get_ep_param,
 139        .ep_connect     = cxgbi_ep_connect,
 140        .ep_poll        = cxgbi_ep_poll,
 141        .ep_disconnect  = cxgbi_ep_disconnect,
 142        /* Error recovery timeout call */
 143        .session_recovery_timedout = iscsi_session_recovery_timedout,
 144};
 145
 146static struct scsi_transport_template *cxgb3i_stt;
 147
 148/*
 149 * CPL (Chelsio Protocol Language) defines a message passing interface between
 150 * the host driver and Chelsio asic.
 151 * The section below implments CPLs that related to iscsi tcp connection
 152 * open/close/abort and data send/receive.
 153 */
 154
 155static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);
 156
 157static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
 158                              const struct l2t_entry *e)
 159{
 160        unsigned int wscale = cxgbi_sock_compute_wscale(csk->rcv_win);
 161        struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;
 162
 163        skb->priority = CPL_PRIORITY_SETUP;
 164
 165        req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 166        OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
 167        req->local_port = csk->saddr.sin_port;
 168        req->peer_port = csk->daddr.sin_port;
 169        req->local_ip = csk->saddr.sin_addr.s_addr;
 170        req->peer_ip = csk->daddr.sin_addr.s_addr;
 171
 172        req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
 173                        V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
 174                        V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
 175        req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
 176                        V_RCV_BUFSIZ(csk->rcv_win >> 10));
 177
 178        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 179                "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
 180                csk, csk->state, csk->flags, csk->atid,
 181                &req->local_ip, ntohs(req->local_port),
 182                &req->peer_ip, ntohs(req->peer_port),
 183                csk->mss_idx, e->idx, e->smt_idx);
 184
 185        l2t_send(csk->cdev->lldev, skb, csk->l2t);
 186}
 187
 188static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
 189{
 190        cxgbi_sock_act_open_req_arp_failure(NULL, skb);
 191}
 192
 193/*
 194 * CPL connection close request: host ->
 195 *
 196 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
 197 * the write queue (i.e., after any unsent txt data).
 198 */
 199static void send_close_req(struct cxgbi_sock *csk)
 200{
 201        struct sk_buff *skb = csk->cpl_close;
 202        struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
 203        unsigned int tid = csk->tid;
 204
 205        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 206                "csk 0x%p,%u,0x%lx,%u.\n",
 207                csk, csk->state, csk->flags, csk->tid);
 208
 209        csk->cpl_close = NULL;
 210        req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
 211        req->wr.wr_lo = htonl(V_WR_TID(tid));
 212        OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 213        req->rsvd = htonl(csk->write_seq);
 214
 215        cxgbi_sock_skb_entail(csk, skb);
 216        if (csk->state >= CTP_ESTABLISHED)
 217                push_tx_frames(csk, 1);
 218}
 219
 220/*
 221 * CPL connection abort request: host ->
 222 *
 223 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
 224 * for the same connection and also that we do not try to send a message
 225 * after the connection has closed.
 226 */
 227static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
 228{
 229        struct cpl_abort_req *req = cplhdr(skb);
 230
 231        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 232                "t3dev 0x%p, tid %u, skb 0x%p.\n",
 233                tdev, GET_TID(req), skb);
 234        req->cmd = CPL_ABORT_NO_RST;
 235        cxgb3_ofld_send(tdev, skb);
 236}
 237
 238static void send_abort_req(struct cxgbi_sock *csk)
 239{
 240        struct sk_buff *skb = csk->cpl_abort_req;
 241        struct cpl_abort_req *req;
 242
 243        if (unlikely(csk->state == CTP_ABORTING || !skb))
 244                return;
 245        cxgbi_sock_set_state(csk, CTP_ABORTING);
 246        cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
 247        /* Purge the send queue so we don't send anything after an abort. */
 248        cxgbi_sock_purge_write_queue(csk);
 249
 250        csk->cpl_abort_req = NULL;
 251        req = (struct cpl_abort_req *)skb->head;
 252        skb->priority = CPL_PRIORITY_DATA;
 253        set_arp_failure_handler(skb, abort_arp_failure);
 254        req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
 255        req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
 256        OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
 257        req->rsvd0 = htonl(csk->snd_nxt);
 258        req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
 259        req->cmd = CPL_ABORT_SEND_RST;
 260
 261        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 262                "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
 263                csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
 264                req->rsvd1);
 265
 266        l2t_send(csk->cdev->lldev, skb, csk->l2t);
 267}
 268
 269/*
 270 * CPL connection abort reply: host ->
 271 *
 272 * Send an ABORT_RPL message in response of the ABORT_REQ received.
 273 */
 274static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
 275{
 276        struct sk_buff *skb = csk->cpl_abort_rpl;
 277        struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
 278
 279        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 280                "csk 0x%p,%u,0x%lx,%u, status %d.\n",
 281                csk, csk->state, csk->flags, csk->tid, rst_status);
 282
 283        csk->cpl_abort_rpl = NULL;
 284        skb->priority = CPL_PRIORITY_DATA;
 285        rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
 286        rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
 287        OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
 288        rpl->cmd = rst_status;
 289        cxgb3_ofld_send(csk->cdev->lldev, skb);
 290}
 291
 292/*
 293 * CPL connection rx data ack: host ->
 294 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
 295 * credits sent.
 296 */
 297static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
 298{
 299        struct sk_buff *skb;
 300        struct cpl_rx_data_ack *req;
 301        u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
 302
 303        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 304                "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
 305                csk, csk->state, csk->flags, csk->tid, credits, dack);
 306
 307        skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
 308        if (!skb) {
 309                pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
 310                return 0;
 311        }
 312        req = (struct cpl_rx_data_ack *)skb->head;
 313        req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 314        OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
 315        req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
 316                                V_RX_CREDITS(credits));
 317        skb->priority = CPL_PRIORITY_ACK;
 318        cxgb3_ofld_send(csk->cdev->lldev, skb);
 319        return credits;
 320}
 321
 322/*
 323 * CPL connection tx data: host ->
 324 *
 325 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
 326 * credits sent.
 327 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
 328 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
 329 */
 330
 331static unsigned int wrlen __read_mostly;
 332static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
 333
 334static void init_wr_tab(unsigned int wr_len)
 335{
 336        int i;
 337
 338        if (skb_wrs[1])         /* already initialized */
 339                return;
 340        for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
 341                int sgl_len = (3 * i) / 2 + (i & 1);
 342
 343                sgl_len += 3;
 344                skb_wrs[i] = (sgl_len <= wr_len
 345                              ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
 346        }
 347        wrlen = wr_len * 8;
 348}
 349
 350static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
 351                                   int len, int req_completion)
 352{
 353        struct tx_data_wr *req;
 354        struct l2t_entry *l2t = csk->l2t;
 355
 356        skb_reset_transport_header(skb);
 357        req = __skb_push(skb, sizeof(*req));
 358        req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
 359                        (req_completion ? F_WR_COMPL : 0));
 360        req->wr_lo = htonl(V_WR_TID(csk->tid));
 361        /* len includes the length of any HW ULP additions */
 362        req->len = htonl(len);
 363        /* V_TX_ULP_SUBMODE sets both the mode and submode */
 364        req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) |
 365                           V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
 366        req->sndseq = htonl(csk->snd_nxt);
 367        req->param = htonl(V_TX_PORT(l2t->smt_idx));
 368
 369        if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
 370                req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
 371                                    V_TX_CPU_IDX(csk->rss_qid));
 372                /* sendbuffer is in units of 32KB. */
 373                req->param |= htonl(V_TX_SNDBUF(csk->snd_win >> 15));
 374                cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 375        }
 376}
 377
 378/**
 379 * push_tx_frames -- start transmit
 380 * @c3cn: the offloaded connection
 381 * @req_completion: request wr_ack or not
 382 *
 383 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
 384 * connection's send queue and sends them on to T3.  Must be called with the
 385 * connection's lock held.  Returns the amount of send buffer space that was
 386 * freed as a result of sending queued data to T3.
 387 */
 388
 389static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
 390{
 391        kfree_skb(skb);
 392}
 393
 394static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
 395{
 396        int total_size = 0;
 397        struct sk_buff *skb;
 398
 399        if (unlikely(csk->state < CTP_ESTABLISHED ||
 400                csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
 401                        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 402                                "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
 403                                csk, csk->state, csk->flags, csk->tid);
 404                return 0;
 405        }
 406
 407        while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
 408                int len = skb->len;     /* length before skb_push */
 409                int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
 410                int wrs_needed = skb_wrs[frags];
 411
 412                if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
 413                        wrs_needed = 1;
 414
 415                WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
 416
 417                if (csk->wr_cred < wrs_needed) {
 418                        log_debug(1 << CXGBI_DBG_PDU_TX,
 419                                "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
 420                                csk, skb->len, skb->data_len, frags,
 421                                wrs_needed, csk->wr_cred);
 422                        break;
 423                }
 424
 425                __skb_unlink(skb, &csk->write_queue);
 426                skb->priority = CPL_PRIORITY_DATA;
 427                skb->csum = wrs_needed; /* remember this until the WR_ACK */
 428                csk->wr_cred -= wrs_needed;
 429                csk->wr_una_cred += wrs_needed;
 430                cxgbi_sock_enqueue_wr(csk, skb);
 431
 432                log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 433                        "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
 434                        "left %u, unack %u.\n",
 435                        csk, skb->len, skb->data_len, frags, skb->csum,
 436                        csk->wr_cred, csk->wr_una_cred);
 437
 438                if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
 439                        if ((req_completion &&
 440                                csk->wr_una_cred == wrs_needed) ||
 441                             csk->wr_una_cred >= csk->wr_max_cred / 2) {
 442                                req_completion = 1;
 443                                csk->wr_una_cred = 0;
 444                        }
 445                        len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
 446                        make_tx_data_wr(csk, skb, len, req_completion);
 447                        csk->snd_nxt += len;
 448                        cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
 449                }
 450                total_size += skb->truesize;
 451                log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 452                        "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
 453                        csk, csk->tid, skb);
 454                set_arp_failure_handler(skb, arp_failure_skb_discard);
 455                l2t_send(csk->cdev->lldev, skb, csk->l2t);
 456        }
 457        return total_size;
 458}
 459
 460/*
 461 * Process a CPL_ACT_ESTABLISH message: -> host
 462 * Updates connection state from an active establish CPL message.  Runs with
 463 * the connection lock held.
 464 */
 465
 466static inline void free_atid(struct cxgbi_sock *csk)
 467{
 468        if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
 469                cxgb3_free_atid(csk->cdev->lldev, csk->atid);
 470                cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
 471                cxgbi_sock_put(csk);
 472        }
 473}
 474
 475static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
 476{
 477        struct cxgbi_sock *csk = ctx;
 478        struct cpl_act_establish *req = cplhdr(skb);
 479        unsigned int tid = GET_TID(req);
 480        unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
 481        u32 rcv_isn = ntohl(req->rcv_isn);      /* real RCV_ISN + 1 */
 482
 483        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 484                "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
 485                atid, atid, csk, csk->state, csk->flags, rcv_isn);
 486
 487        cxgbi_sock_get(csk);
 488        cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
 489        csk->tid = tid;
 490        cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);
 491
 492        free_atid(csk);
 493
 494        csk->rss_qid = G_QNUM(ntohs(skb->csum));
 495
 496        spin_lock_bh(&csk->lock);
 497        if (csk->retry_timer.function) {
 498                del_timer(&csk->retry_timer);
 499                csk->retry_timer.function = NULL;
 500        }
 501
 502        if (unlikely(csk->state != CTP_ACTIVE_OPEN))
 503                pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
 504                        csk, csk->state, csk->flags, csk->tid);
 505
 506        csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
 507        if (csk->rcv_win > (M_RCV_BUFSIZ << 10))
 508                csk->rcv_wup -= csk->rcv_win - (M_RCV_BUFSIZ << 10);
 509
 510        cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
 511
 512        if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
 513                /* upper layer has requested closing */
 514                send_abort_req(csk);
 515        else {
 516                if (skb_queue_len(&csk->write_queue))
 517                        push_tx_frames(csk, 1);
 518                cxgbi_conn_tx_open(csk);
 519        }
 520
 521        spin_unlock_bh(&csk->lock);
 522        __kfree_skb(skb);
 523        return 0;
 524}
 525
 526/*
 527 * Process a CPL_ACT_OPEN_RPL message: -> host
 528 * Handle active open failures.
 529 */
 530static int act_open_rpl_status_to_errno(int status)
 531{
 532        switch (status) {
 533        case CPL_ERR_CONN_RESET:
 534                return -ECONNREFUSED;
 535        case CPL_ERR_ARP_MISS:
 536                return -EHOSTUNREACH;
 537        case CPL_ERR_CONN_TIMEDOUT:
 538                return -ETIMEDOUT;
 539        case CPL_ERR_TCAM_FULL:
 540                return -ENOMEM;
 541        case CPL_ERR_CONN_EXIST:
 542                return -EADDRINUSE;
 543        default:
 544                return -EIO;
 545        }
 546}
 547
 548static void act_open_retry_timer(struct timer_list *t)
 549{
 550        struct cxgbi_sock *csk = from_timer(csk, t, retry_timer);
 551        struct sk_buff *skb;
 552
 553        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 554                "csk 0x%p,%u,0x%lx,%u.\n",
 555                csk, csk->state, csk->flags, csk->tid);
 556
 557        cxgbi_sock_get(csk);
 558        spin_lock_bh(&csk->lock);
 559        skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
 560        if (!skb)
 561                cxgbi_sock_fail_act_open(csk, -ENOMEM);
 562        else {
 563                skb->sk = (struct sock *)csk;
 564                set_arp_failure_handler(skb, act_open_arp_failure);
 565                send_act_open_req(csk, skb, csk->l2t);
 566        }
 567        spin_unlock_bh(&csk->lock);
 568        cxgbi_sock_put(csk);
 569}
 570
 571static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
 572{
 573        struct cxgbi_sock *csk = ctx;
 574        struct cpl_act_open_rpl *rpl = cplhdr(skb);
 575
 576        pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
 577                csk, csk->state, csk->flags, csk->atid, rpl->status,
 578                &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
 579                &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
 580
 581        if (rpl->status != CPL_ERR_TCAM_FULL &&
 582            rpl->status != CPL_ERR_CONN_EXIST &&
 583            rpl->status != CPL_ERR_ARP_MISS)
 584                cxgb3_queue_tid_release(tdev, GET_TID(rpl));
 585
 586        cxgbi_sock_get(csk);
 587        spin_lock_bh(&csk->lock);
 588        if (rpl->status == CPL_ERR_CONN_EXIST &&
 589            csk->retry_timer.function != act_open_retry_timer) {
 590                csk->retry_timer.function = act_open_retry_timer;
 591                mod_timer(&csk->retry_timer, jiffies + HZ / 2);
 592        } else
 593                cxgbi_sock_fail_act_open(csk,
 594                                act_open_rpl_status_to_errno(rpl->status));
 595
 596        spin_unlock_bh(&csk->lock);
 597        cxgbi_sock_put(csk);
 598        __kfree_skb(skb);
 599        return 0;
 600}
 601
 602/*
 603 * Process PEER_CLOSE CPL messages: -> host
 604 * Handle peer FIN.
 605 */
 606static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 607{
 608        struct cxgbi_sock *csk = ctx;
 609
 610        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 611                "csk 0x%p,%u,0x%lx,%u.\n",
 612                csk, csk->state, csk->flags, csk->tid);
 613
 614        cxgbi_sock_rcv_peer_close(csk);
 615        __kfree_skb(skb);
 616        return 0;
 617}
 618
 619/*
 620 * Process CLOSE_CONN_RPL CPL message: -> host
 621 * Process a peer ACK to our FIN.
 622 */
 623static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
 624                            void *ctx)
 625{
 626        struct cxgbi_sock *csk = ctx;
 627        struct cpl_close_con_rpl *rpl = cplhdr(skb);
 628
 629        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 630                "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
 631                csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));
 632
 633        cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
 634        __kfree_skb(skb);
 635        return 0;
 636}
 637
 638/*
 639 * Process ABORT_REQ_RSS CPL message: -> host
 640 * Process abort requests.  If we are waiting for an ABORT_RPL we ignore this
 641 * request except that we need to reply to it.
 642 */
 643
 644static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
 645                                 int *need_rst)
 646{
 647        switch (abort_reason) {
 648        case CPL_ERR_BAD_SYN: /* fall through */
 649        case CPL_ERR_CONN_RESET:
 650                return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
 651        case CPL_ERR_XMIT_TIMEDOUT:
 652        case CPL_ERR_PERSIST_TIMEDOUT:
 653        case CPL_ERR_FINWAIT2_TIMEDOUT:
 654        case CPL_ERR_KEEPALIVE_TIMEDOUT:
 655                return -ETIMEDOUT;
 656        default:
 657                return -EIO;
 658        }
 659}
 660
 661static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 662{
 663        const struct cpl_abort_req_rss *req = cplhdr(skb);
 664        struct cxgbi_sock *csk = ctx;
 665        int rst_status = CPL_ABORT_NO_RST;
 666
 667        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 668                "csk 0x%p,%u,0x%lx,%u.\n",
 669                csk, csk->state, csk->flags, csk->tid);
 670
 671        if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
 672            req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
 673                goto done;
 674        }
 675
 676        cxgbi_sock_get(csk);
 677        spin_lock_bh(&csk->lock);
 678
 679        if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
 680                cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
 681                cxgbi_sock_set_state(csk, CTP_ABORTING);
 682                goto out;
 683        }
 684
 685        cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
 686        send_abort_rpl(csk, rst_status);
 687
 688        if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
 689                csk->err = abort_status_to_errno(csk, req->status, &rst_status);
 690                cxgbi_sock_closed(csk);
 691        }
 692
 693out:
 694        spin_unlock_bh(&csk->lock);
 695        cxgbi_sock_put(csk);
 696done:
 697        __kfree_skb(skb);
 698        return 0;
 699}
 700
 701/*
 702 * Process ABORT_RPL_RSS CPL message: -> host
 703 * Process abort replies.  We only process these messages if we anticipate
 704 * them as the coordination between SW and HW in this area is somewhat lacking
 705 * and sometimes we get ABORT_RPLs after we are done with the connection that
 706 * originated the ABORT_REQ.
 707 */
 708static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 709{
 710        struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
 711        struct cxgbi_sock *csk = ctx;
 712
 713        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 714                "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
 715                rpl->status, csk, csk ? csk->state : 0,
 716                csk ? csk->flags : 0UL);
 717        /*
 718         * Ignore replies to post-close aborts indicating that the abort was
 719         * requested too late.  These connections are terminated when we get
 720         * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
 721         * arrives the TID is either no longer used or it has been recycled.
 722         */
 723        if (rpl->status == CPL_ERR_ABORT_FAILED)
 724                goto rel_skb;
 725        /*
 726         * Sometimes we've already closed the connection, e.g., a post-close
 727         * abort races with ABORT_REQ_RSS, the latter frees the connection
 728         * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
 729         * but FW turns the ABORT_REQ into a regular one and so we get
 730         * ABORT_RPL_RSS with status 0 and no connection.
 731         */
 732        if (csk)
 733                cxgbi_sock_rcv_abort_rpl(csk);
 734rel_skb:
 735        __kfree_skb(skb);
 736        return 0;
 737}
 738
 739/*
 740 * Process RX_ISCSI_HDR CPL message: -> host
 741 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
 742 * follow after the bhs.
 743 */
 744static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
 745{
 746        struct cxgbi_sock *csk = ctx;
 747        struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
 748        struct cpl_iscsi_hdr_norss data_cpl;
 749        struct cpl_rx_data_ddp_norss ddp_cpl;
 750        unsigned int hdr_len, data_len, status;
 751        unsigned int len;
 752        int err;
 753
 754        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 755                "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
 756                csk, csk->state, csk->flags, csk->tid, skb, skb->len);
 757
 758        spin_lock_bh(&csk->lock);
 759
 760        if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
 761                log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 762                        "csk 0x%p,%u,0x%lx,%u, bad state.\n",
 763                        csk, csk->state, csk->flags, csk->tid);
 764                if (csk->state != CTP_ABORTING)
 765                        goto abort_conn;
 766                else
 767                        goto discard;
 768        }
 769
 770        cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
 771        cxgbi_skcb_flags(skb) = 0;
 772
 773        skb_reset_transport_header(skb);
 774        __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
 775
 776        len = hdr_len = ntohs(hdr_cpl->len);
 777        /* msg coalesce is off or not enough data received */
 778        if (skb->len <= hdr_len) {
 779                pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
 780                        csk->cdev->ports[csk->port_id]->name, csk->tid,
 781                        skb->len, hdr_len);
 782                goto abort_conn;
 783        }
 784        cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);
 785
 786        err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
 787                            sizeof(ddp_cpl));
 788        if (err < 0) {
 789                pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
 790                        csk->cdev->ports[csk->port_id]->name, csk->tid,
 791                        skb->len, sizeof(ddp_cpl), err);
 792                goto abort_conn;
 793        }
 794
 795        cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
 796        cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
 797        cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
 798        status = ntohl(ddp_cpl.ddp_status);
 799
 800        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 801                "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
 802                csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);
 803
 804        if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
 805                cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
 806        if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
 807                cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
 808        if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
 809                cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);
 810
 811        if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
 812                err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
 813                if (err < 0) {
 814                        pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
 815                                csk->cdev->ports[csk->port_id]->name,
 816                                csk->tid, sizeof(data_cpl), skb->len, err);
 817                        goto abort_conn;
 818                }
 819                data_len = ntohs(data_cpl.len);
 820                log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
 821                        "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
 822                        skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
 823                len += sizeof(data_cpl) + data_len;
 824        } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
 825                cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);
 826
 827        csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
 828        __pskb_trim(skb, len);
 829        __skb_queue_tail(&csk->receive_queue, skb);
 830        cxgbi_conn_pdu_ready(csk);
 831
 832        spin_unlock_bh(&csk->lock);
 833        return 0;
 834
 835abort_conn:
 836        send_abort_req(csk);
 837discard:
 838        spin_unlock_bh(&csk->lock);
 839        __kfree_skb(skb);
 840        return 0;
 841}
 842
 843/*
 844 * Process TX_DATA_ACK CPL messages: -> host
 845 * Process an acknowledgment of WR completion.  Advance snd_una and send the
 846 * next batch of work requests from the write queue.
 847 */
 848static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 849{
 850        struct cxgbi_sock *csk = ctx;
 851        struct cpl_wr_ack *hdr = cplhdr(skb);
 852
 853        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 854                "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
 855                csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));
 856
 857        cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
 858        __kfree_skb(skb);
 859        return 0;
 860}
 861
 862/*
 863 * for each connection, pre-allocate skbs needed for close/abort requests. So
 864 * that we can service the request right away.
 865 */
 866static int alloc_cpls(struct cxgbi_sock *csk)
 867{
 868        csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,
 869                                        GFP_KERNEL);
 870        if (!csk->cpl_close)
 871                return -ENOMEM;
 872        csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,
 873                                        GFP_KERNEL);
 874        if (!csk->cpl_abort_req)
 875                goto free_cpl_skbs;
 876
 877        csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,
 878                                        GFP_KERNEL);
 879        if (!csk->cpl_abort_rpl)
 880                goto free_cpl_skbs;
 881
 882        return 0;
 883
 884free_cpl_skbs:
 885        cxgbi_sock_free_cpl_skbs(csk);
 886        return -ENOMEM;
 887}
 888
 889/**
 890 * release_offload_resources - release offload resource
 891 * @c3cn: the offloaded iscsi tcp connection.
 892 * Release resources held by an offload connection (TID, L2T entry, etc.)
 893 */
 894static void l2t_put(struct cxgbi_sock *csk)
 895{
 896        struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
 897
 898        if (csk->l2t) {
 899                l2t_release(t3dev, csk->l2t);
 900                csk->l2t = NULL;
 901                cxgbi_sock_put(csk);
 902        }
 903}
 904
 905static void release_offload_resources(struct cxgbi_sock *csk)
 906{
 907        struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
 908
 909        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 910                "csk 0x%p,%u,0x%lx,%u.\n",
 911                csk, csk->state, csk->flags, csk->tid);
 912
 913        csk->rss_qid = 0;
 914        cxgbi_sock_free_cpl_skbs(csk);
 915
 916        if (csk->wr_cred != csk->wr_max_cred) {
 917                cxgbi_sock_purge_wr_queue(csk);
 918                cxgbi_sock_reset_wr_list(csk);
 919        }
 920        l2t_put(csk);
 921        if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
 922                free_atid(csk);
 923        else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
 924                cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);
 925                cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
 926                cxgbi_sock_put(csk);
 927        }
 928        csk->dst = NULL;
 929        csk->cdev = NULL;
 930}
 931
 932static void update_address(struct cxgbi_hba *chba)
 933{
 934        if (chba->ipv4addr) {
 935                if (chba->vdev &&
 936                    chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {
 937                        cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);
 938                        cxgb3i_set_private_ipv4addr(chba->ndev, 0);
 939                        pr_info("%s set %pI4.\n",
 940                                chba->vdev->name, &chba->ipv4addr);
 941                } else if (chba->ipv4addr !=
 942                                cxgb3i_get_private_ipv4addr(chba->ndev)) {
 943                        cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);
 944                        pr_info("%s set %pI4.\n",
 945                                chba->ndev->name, &chba->ipv4addr);
 946                }
 947        } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {
 948                if (chba->vdev)
 949                        cxgb3i_set_private_ipv4addr(chba->vdev, 0);
 950                cxgb3i_set_private_ipv4addr(chba->ndev, 0);
 951        }
 952}
 953
 954static int init_act_open(struct cxgbi_sock *csk)
 955{
 956        struct dst_entry *dst = csk->dst;
 957        struct cxgbi_device *cdev = csk->cdev;
 958        struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
 959        struct net_device *ndev = cdev->ports[csk->port_id];
 960        struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
 961        struct sk_buff *skb = NULL;
 962
 963        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 964                "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
 965
 966        update_address(chba);
 967        if (chba->ipv4addr)
 968                csk->saddr.sin_addr.s_addr = chba->ipv4addr;
 969
 970        csk->rss_qid = 0;
 971        csk->l2t = t3_l2t_get(t3dev, dst, ndev,
 972                              &csk->daddr.sin_addr.s_addr);
 973        if (!csk->l2t) {
 974                pr_err("NO l2t available.\n");
 975                return -EINVAL;
 976        }
 977        cxgbi_sock_get(csk);
 978
 979        csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
 980        if (csk->atid < 0) {
 981                pr_err("NO atid available.\n");
 982                goto rel_resource;
 983        }
 984        cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
 985        cxgbi_sock_get(csk);
 986
 987        skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
 988        if (!skb)
 989                goto rel_resource;
 990        skb->sk = (struct sock *)csk;
 991        set_arp_failure_handler(skb, act_open_arp_failure);
 992        csk->snd_win = cxgb3i_snd_win;
 993        csk->rcv_win = cxgb3i_rcv_win;
 994
 995        csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
 996        csk->wr_una_cred = 0;
 997        csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
 998        cxgbi_sock_reset_wr_list(csk);
 999        csk->err = 0;
1000
1001        log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1002                "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
1003                csk, csk->state, csk->flags,
1004                &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
1005                &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
1006
1007        cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
1008        send_act_open_req(csk, skb, csk->l2t);
1009        return 0;
1010
1011rel_resource:
1012        if (skb)
1013                __kfree_skb(skb);
1014        return -EINVAL;
1015}
1016
1017cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
1018        [CPL_ACT_ESTABLISH] = do_act_establish,
1019        [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1020        [CPL_PEER_CLOSE] = do_peer_close,
1021        [CPL_ABORT_REQ_RSS] = do_abort_req,
1022        [CPL_ABORT_RPL_RSS] = do_abort_rpl,
1023        [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1024        [CPL_TX_DMA_ACK] = do_wr_ack,
1025        [CPL_ISCSI_HDR] = do_iscsi_hdr,
1026};
1027
1028/**
1029 * cxgb3i_ofld_init - allocate and initialize resources for each adapter found
1030 * @cdev:       cxgbi adapter
1031 */
1032static int cxgb3i_ofld_init(struct cxgbi_device *cdev)
1033{
1034        struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
1035        struct adap_ports port;
1036        struct ofld_page_info rx_page_info;
1037        unsigned int wr_len;
1038        int rc;
1039
1040        if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
1041            t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
1042            t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
1043                pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
1044                return -EINVAL;
1045        }
1046
1047        if (cxgb3i_max_connect > CXGBI_MAX_CONN)
1048                cxgb3i_max_connect = CXGBI_MAX_CONN;
1049
1050        rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,
1051                                        cxgb3i_max_connect);
1052        if (rc < 0)
1053                return rc;
1054
1055        init_wr_tab(wr_len);
1056        cdev->csk_release_offload_resources = release_offload_resources;
1057        cdev->csk_push_tx_frames = push_tx_frames;
1058        cdev->csk_send_abort_req = send_abort_req;
1059        cdev->csk_send_close_req = send_close_req;
1060        cdev->csk_send_rx_credits = send_rx_credits;
1061        cdev->csk_alloc_cpls = alloc_cpls;
1062        cdev->csk_init_act_open = init_act_open;
1063
1064        pr_info("cdev 0x%p, offload up, added.\n", cdev);
1065        return 0;
1066}
1067
1068/*
1069 * functions to program the pagepod in h/w
1070 */
1071static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
1072{
1073        struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
1074
1075        memset(req, 0, sizeof(*req));
1076
1077        req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
1078        req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
1079                                   V_ULPTX_CMD(ULP_MEM_WRITE));
1080        req->len = htonl(V_ULP_MEMIO_DATA_LEN(IPPOD_SIZE >> 5) |
1081                         V_ULPTX_NFLITS((IPPOD_SIZE >> 3) + 1));
1082}
1083
1084static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev)
1085{
1086        return ((struct t3cdev *)cdev->lldev)->ulp_iscsi;
1087}
1088
1089static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,
1090                       struct cxgbi_task_tag_info *ttinfo)
1091{
1092        unsigned int idx = ttinfo->idx;
1093        unsigned int npods = ttinfo->npods;
1094        struct scatterlist *sg = ttinfo->sgl;
1095        struct cxgbi_pagepod *ppod;
1096        struct ulp_mem_io *req;
1097        unsigned int sg_off;
1098        unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
1099        int i;
1100
1101        for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
1102                struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1103                                               IPPOD_SIZE, 0, GFP_ATOMIC);
1104
1105                if (!skb)
1106                        return -ENOMEM;
1107                ulp_mem_io_set_hdr(skb, pm_addr);
1108                req = (struct ulp_mem_io *)skb->head;
1109                ppod = (struct cxgbi_pagepod *)(req + 1);
1110                sg_off = i * PPOD_PAGES_MAX;
1111                cxgbi_ddp_set_one_ppod(ppod, ttinfo, &sg,
1112                                       &sg_off);
1113                skb->priority = CPL_PRIORITY_CONTROL;
1114                cxgb3_ofld_send(ppm->lldev, skb);
1115        }
1116        return 0;
1117}
1118
1119static void ddp_clear_map(struct cxgbi_device *cdev, struct cxgbi_ppm *ppm,
1120                          struct cxgbi_task_tag_info *ttinfo)
1121{
1122        unsigned int idx = ttinfo->idx;
1123        unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
1124        unsigned int npods = ttinfo->npods;
1125        int i;
1126
1127        log_debug(1 << CXGBI_DBG_DDP,
1128                  "cdev 0x%p, clear idx %u, npods %u.\n",
1129                  cdev, idx, npods);
1130
1131        for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {
1132                struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1133                                               IPPOD_SIZE, 0, GFP_ATOMIC);
1134
1135                if (!skb) {
1136                        pr_err("cdev 0x%p, clear ddp, %u,%d/%u, skb OOM.\n",
1137                               cdev, idx, i, npods);
1138                        continue;
1139                }
1140                ulp_mem_io_set_hdr(skb, pm_addr);
1141                skb->priority = CPL_PRIORITY_CONTROL;
1142                cxgb3_ofld_send(ppm->lldev, skb);
1143        }
1144}
1145
1146static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
1147                                       unsigned int tid, int pg_idx, bool reply)
1148{
1149        struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1150                                        GFP_KERNEL);
1151        struct cpl_set_tcb_field *req;
1152        u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
1153
1154        log_debug(1 << CXGBI_DBG_DDP,
1155                "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
1156        if (!skb)
1157                return -ENOMEM;
1158
1159        /* set up ulp submode and page size */
1160        req = (struct cpl_set_tcb_field *)skb->head;
1161        req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1162        OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1163        req->reply = V_NO_REPLY(reply ? 0 : 1);
1164        req->cpu_idx = 0;
1165        req->word = htons(31);
1166        req->mask = cpu_to_be64(0xF0000000);
1167        req->val = cpu_to_be64(val << 28);
1168        skb->priority = CPL_PRIORITY_CONTROL;
1169
1170        cxgb3_ofld_send(csk->cdev->lldev, skb);
1171        return 0;
1172}
1173
1174/**
1175 * cxgb3i_setup_conn_digest - setup conn. digest setting
1176 * @csk: cxgb tcp socket
1177 * @tid: connection id
1178 * @hcrc: header digest enabled
1179 * @dcrc: data digest enabled
1180 * @reply: request reply from h/w
1181 * set up the iscsi digest settings for a connection identified by tid
1182 */
1183static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1184                             int hcrc, int dcrc, int reply)
1185{
1186        struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1187                                        GFP_KERNEL);
1188        struct cpl_set_tcb_field *req;
1189        u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
1190
1191        log_debug(1 << CXGBI_DBG_DDP,
1192                "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
1193        if (!skb)
1194                return -ENOMEM;
1195
1196        /* set up ulp submode and page size */
1197        req = (struct cpl_set_tcb_field *)skb->head;
1198        req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1199        OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1200        req->reply = V_NO_REPLY(reply ? 0 : 1);
1201        req->cpu_idx = 0;
1202        req->word = htons(31);
1203        req->mask = cpu_to_be64(0x0F000000);
1204        req->val = cpu_to_be64(val << 24);
1205        skb->priority = CPL_PRIORITY_CONTROL;
1206
1207        cxgb3_ofld_send(csk->cdev->lldev, skb);
1208        return 0;
1209}
1210
1211/**
1212 * cxgb3i_ddp_init - initialize the cxgb3 adapter's ddp resource
1213 * @cdev: cxgb3i adapter
1214 * initialize the ddp pagepod manager for a given adapter
1215 */
1216static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
1217{
1218        struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
1219        struct net_device *ndev = cdev->ports[0];
1220        struct cxgbi_tag_format tformat;
1221        unsigned int ppmax, tagmask = 0;
1222        struct ulp_iscsi_info uinfo;
1223        int i, err;
1224
1225        err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
1226        if (err < 0) {
1227                pr_err("%s, failed to get iscsi param %d.\n",
1228                       ndev->name, err);
1229                return err;
1230        }
1231        if (uinfo.llimit >= uinfo.ulimit) {
1232                pr_warn("T3 %s, iscsi NOT enabled %u ~ %u!\n",
1233                        ndev->name, uinfo.llimit, uinfo.ulimit);
1234                return -EACCES;
1235        }
1236
1237        ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;
1238        tagmask = cxgbi_tagmask_set(ppmax);
1239
1240        pr_info("T3 %s: 0x%x~0x%x, 0x%x, tagmask 0x%x -> 0x%x.\n",
1241                ndev->name, uinfo.llimit, uinfo.ulimit, ppmax, uinfo.tagmask,
1242                tagmask);
1243
1244        memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
1245        for (i = 0; i < 4; i++)
1246                tformat.pgsz_order[i] = uinfo.pgsz_factor[i];
1247        cxgbi_tagmask_check(tagmask, &tformat);
1248
1249        cxgbi_ddp_ppm_setup(&tdev->ulp_iscsi, cdev, &tformat, ppmax,
1250                            uinfo.llimit, uinfo.llimit, 0);
1251        if (!(cdev->flags & CXGBI_FLAG_DDP_OFF)) {
1252                uinfo.tagmask = tagmask;
1253                uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);
1254
1255                err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
1256                if (err < 0) {
1257                        pr_err("T3 %s fail to set iscsi param %d.\n",
1258                               ndev->name, err);
1259                        cdev->flags |= CXGBI_FLAG_DDP_OFF;
1260                }
1261                err = 0;
1262        }
1263
1264        cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1265        cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1266        cdev->csk_ddp_set_map = ddp_set_map;
1267        cdev->csk_ddp_clear_map = ddp_clear_map;
1268        cdev->cdev2ppm = cdev2ppm;
1269        cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1270                                  uinfo.max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
1271        cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1272                                  uinfo.max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
1273
1274        return 0;
1275}
1276
1277static void cxgb3i_dev_close(struct t3cdev *t3dev)
1278{
1279        struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1280
1281        if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
1282                pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
1283                return;
1284        }
1285
1286        cxgbi_device_unregister(cdev);
1287}
1288
1289/**
1290 * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
1291 * @t3dev: t3cdev adapter
1292 */
1293static void cxgb3i_dev_open(struct t3cdev *t3dev)
1294{
1295        struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1296        struct adapter *adapter = tdev2adap(t3dev);
1297        int i, err;
1298
1299        if (cdev) {
1300                pr_info("0x%p, updating.\n", cdev);
1301                return;
1302        }
1303
1304        cdev = cxgbi_device_register(0, adapter->params.nports);
1305        if (!cdev) {
1306                pr_warn("device 0x%p register failed.\n", t3dev);
1307                return;
1308        }
1309
1310        cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
1311        cdev->lldev = t3dev;
1312        cdev->pdev = adapter->pdev;
1313        cdev->ports = adapter->port;
1314        cdev->nports = adapter->params.nports;
1315        cdev->mtus = adapter->params.mtus;
1316        cdev->nmtus = NMTUS;
1317        cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
1318        cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
1319        cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
1320        cdev->itp = &cxgb3i_iscsi_transport;
1321
1322        err = cxgb3i_ddp_init(cdev);
1323        if (err) {
1324                pr_info("0x%p ddp init failed\n", cdev);
1325                goto err_out;
1326        }
1327
1328        err = cxgb3i_ofld_init(cdev);
1329        if (err) {
1330                pr_info("0x%p offload init failed\n", cdev);
1331                goto err_out;
1332        }
1333
1334        err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
1335                                &cxgb3i_host_template, cxgb3i_stt);
1336        if (err)
1337                goto err_out;
1338
1339        for (i = 0; i < cdev->nports; i++)
1340                cdev->hbas[i]->ipv4addr =
1341                        cxgb3i_get_private_ipv4addr(cdev->ports[i]);
1342
1343        pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
1344                cdev, cdev ? cdev->flags : 0, t3dev, err);
1345        return;
1346
1347err_out:
1348        cxgbi_device_unregister(cdev);
1349}
1350
1351static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
1352{
1353        struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1354
1355        log_debug(1 << CXGBI_DBG_TOE,
1356                "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
1357                t3dev, cdev, event, port);
1358        if (!cdev)
1359                return;
1360
1361        switch (event) {
1362        case OFFLOAD_STATUS_DOWN:
1363                cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
1364                break;
1365        case OFFLOAD_STATUS_UP:
1366                cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
1367                break;
1368        }
1369}
1370
1371/**
1372 * cxgb3i_init_module - module init entry point
1373 *
1374 * initialize any driver wide global data structures and register itself
1375 *      with the cxgb3 module
1376 */
1377static int __init cxgb3i_init_module(void)
1378{
1379        int rc;
1380
1381        printk(KERN_INFO "%s", version);
1382
1383        rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1384        if (rc < 0)
1385                return rc;
1386
1387        cxgb3_register_client(&t3_client);
1388        return 0;
1389}
1390
1391/**
1392 * cxgb3i_exit_module - module cleanup/exit entry point
1393 *
1394 * go through the driver hba list and for each hba, release any resource held.
1395 *      and unregisters iscsi transport and the cxgb3 module
1396 */
1397static void __exit cxgb3i_exit_module(void)
1398{
1399        cxgb3_unregister_client(&t3_client);
1400        cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
1401        cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1402}
1403
1404module_init(cxgb3i_init_module);
1405module_exit(cxgb3i_exit_module);
1406