linux/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *      - Redistributions in binary form must reproduce the above
  18 *        copyright notice, this list of conditions and the following
  19 *        disclaimer in the documentation and/or other materials
  20 *        provided with the distribution.
  21 *
  22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  29 * SOFTWARE.
  30 */
  31#ifndef __IW_CXGB4_H__
  32#define __IW_CXGB4_H__
  33
  34#include <linux/mutex.h>
  35#include <linux/list.h>
  36#include <linux/spinlock.h>
  37#include <linux/xarray.h>
  38#include <linux/completion.h>
  39#include <linux/netdevice.h>
  40#include <linux/sched/mm.h>
  41#include <linux/pci.h>
  42#include <linux/dma-mapping.h>
  43#include <linux/inet.h>
  44#include <linux/wait.h>
  45#include <linux/kref.h>
  46#include <linux/timer.h>
  47#include <linux/io.h>
  48#include <linux/workqueue.h>
  49
  50#include <asm/byteorder.h>
  51
  52#include <net/net_namespace.h>
  53
  54#include <rdma/ib_verbs.h>
  55#include <rdma/iw_cm.h>
  56#include <rdma/rdma_netlink.h>
  57#include <rdma/iw_portmap.h>
  58#include <rdma/restrack.h>
  59
  60#include "cxgb4.h"
  61#include "cxgb4_uld.h"
  62#include "l2t.h"
  63#include <rdma/cxgb4-abi.h>
  64
  65#define DRV_NAME "iw_cxgb4"
  66#define MOD DRV_NAME ":"
  67
  68#ifdef pr_fmt
  69#undef pr_fmt
  70#endif
  71
  72#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  73
  74#include "t4.h"
  75
  76#define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->pbl.start)
  77#define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->lldi.vr->rq.start)
  78
  79static inline void *cplhdr(struct sk_buff *skb)
  80{
  81        return skb->data;
  82}
  83
  84#define C4IW_ID_TABLE_F_RANDOM 1       /* Pseudo-randomize the id's returned */
  85#define C4IW_ID_TABLE_F_EMPTY  2       /* Table is initially empty */
  86
  87struct c4iw_id_table {
  88        u32 flags;
  89        u32 start;              /* logical minimal id */
  90        u32 last;               /* hint for find */
  91        u32 max;
  92        spinlock_t lock;
  93        unsigned long *table;
  94};
  95
  96struct c4iw_resource {
  97        struct c4iw_id_table tpt_table;
  98        struct c4iw_id_table qid_table;
  99        struct c4iw_id_table pdid_table;
 100        struct c4iw_id_table srq_table;
 101};
 102
 103struct c4iw_qid_list {
 104        struct list_head entry;
 105        u32 qid;
 106};
 107
 108struct c4iw_dev_ucontext {
 109        struct list_head qpids;
 110        struct list_head cqids;
 111        struct mutex lock;
 112        struct kref kref;
 113};
 114
 115enum c4iw_rdev_flags {
 116        T4_FATAL_ERROR = (1<<0),
 117        T4_STATUS_PAGE_DISABLED = (1<<1),
 118};
 119
 120struct c4iw_stat {
 121        u64 total;
 122        u64 cur;
 123        u64 max;
 124        u64 fail;
 125};
 126
 127struct c4iw_stats {
 128        struct mutex lock;
 129        struct c4iw_stat qid;
 130        struct c4iw_stat pd;
 131        struct c4iw_stat stag;
 132        struct c4iw_stat pbl;
 133        struct c4iw_stat rqt;
 134        struct c4iw_stat srqt;
 135        struct c4iw_stat srq;
 136        struct c4iw_stat ocqp;
 137        u64  db_full;
 138        u64  db_empty;
 139        u64  db_drop;
 140        u64  db_state_transitions;
 141        u64  db_fc_interruptions;
 142        u64  tcam_full;
 143        u64  act_ofld_conn_fails;
 144        u64  pas_ofld_conn_fails;
 145        u64  neg_adv;
 146};
 147
 148struct c4iw_hw_queue {
 149        int t4_eq_status_entries;
 150        int t4_max_eq_size;
 151        int t4_max_iq_size;
 152        int t4_max_rq_size;
 153        int t4_max_sq_size;
 154        int t4_max_qp_depth;
 155        int t4_max_cq_depth;
 156        int t4_stat_len;
 157};
 158
 159struct wr_log_entry {
 160        ktime_t post_host_time;
 161        ktime_t poll_host_time;
 162        u64 post_sge_ts;
 163        u64 cqe_sge_ts;
 164        u64 poll_sge_ts;
 165        u16 qid;
 166        u16 wr_id;
 167        u8 opcode;
 168        u8 valid;
 169};
 170
 171struct c4iw_rdev {
 172        struct c4iw_resource resource;
 173        u32 qpmask;
 174        u32 cqmask;
 175        struct c4iw_dev_ucontext uctx;
 176        struct gen_pool *pbl_pool;
 177        struct gen_pool *rqt_pool;
 178        struct gen_pool *ocqp_pool;
 179        u32 flags;
 180        struct cxgb4_lld_info lldi;
 181        unsigned long bar2_pa;
 182        void __iomem *bar2_kva;
 183        unsigned long oc_mw_pa;
 184        void __iomem *oc_mw_kva;
 185        struct c4iw_stats stats;
 186        struct c4iw_hw_queue hw_queue;
 187        struct t4_dev_status_page *status_page;
 188        atomic_t wr_log_idx;
 189        struct wr_log_entry *wr_log;
 190        int wr_log_size;
 191        struct workqueue_struct *free_workq;
 192        struct completion rqt_compl;
 193        struct completion pbl_compl;
 194        struct kref rqt_kref;
 195        struct kref pbl_kref;
 196};
 197
 198static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
 199{
 200        return rdev->flags & T4_FATAL_ERROR;
 201}
 202
 203static inline int c4iw_num_stags(struct c4iw_rdev *rdev)
 204{
 205        return (int)(rdev->lldi.vr->stag.size >> 5);
 206}
 207
 208#define C4IW_WR_TO (60*HZ)
 209
 210struct c4iw_wr_wait {
 211        struct completion completion;
 212        int ret;
 213        struct kref kref;
 214};
 215
 216void _c4iw_free_wr_wait(struct kref *kref);
 217
 218static inline void c4iw_put_wr_wait(struct c4iw_wr_wait *wr_waitp)
 219{
 220        pr_debug("wr_wait %p ref before put %u\n", wr_waitp,
 221                 kref_read(&wr_waitp->kref));
 222        WARN_ON(kref_read(&wr_waitp->kref) == 0);
 223        kref_put(&wr_waitp->kref, _c4iw_free_wr_wait);
 224}
 225
 226static inline void c4iw_get_wr_wait(struct c4iw_wr_wait *wr_waitp)
 227{
 228        pr_debug("wr_wait %p ref before get %u\n", wr_waitp,
 229                 kref_read(&wr_waitp->kref));
 230        WARN_ON(kref_read(&wr_waitp->kref) == 0);
 231        kref_get(&wr_waitp->kref);
 232}
 233
 234static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
 235{
 236        wr_waitp->ret = 0;
 237        init_completion(&wr_waitp->completion);
 238}
 239
 240static inline void _c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret,
 241                                 bool deref)
 242{
 243        wr_waitp->ret = ret;
 244        complete(&wr_waitp->completion);
 245        if (deref)
 246                c4iw_put_wr_wait(wr_waitp);
 247}
 248
 249static inline void c4iw_wake_up_noref(struct c4iw_wr_wait *wr_waitp, int ret)
 250{
 251        _c4iw_wake_up(wr_waitp, ret, false);
 252}
 253
 254static inline void c4iw_wake_up_deref(struct c4iw_wr_wait *wr_waitp, int ret)
 255{
 256        _c4iw_wake_up(wr_waitp, ret, true);
 257}
 258
 259static inline int c4iw_wait_for_reply(struct c4iw_rdev *rdev,
 260                                 struct c4iw_wr_wait *wr_waitp,
 261                                 u32 hwtid, u32 qpid,
 262                                 const char *func)
 263{
 264        int ret;
 265
 266        if (c4iw_fatal_error(rdev)) {
 267                wr_waitp->ret = -EIO;
 268                goto out;
 269        }
 270
 271        ret = wait_for_completion_timeout(&wr_waitp->completion, C4IW_WR_TO);
 272        if (!ret) {
 273                pr_err("%s - Device %s not responding (disabling device) - tid %u qpid %u\n",
 274                       func, pci_name(rdev->lldi.pdev), hwtid, qpid);
 275                rdev->flags |= T4_FATAL_ERROR;
 276                wr_waitp->ret = -EIO;
 277                goto out;
 278        }
 279        if (wr_waitp->ret)
 280                pr_debug("%s: FW reply %d tid %u qpid %u\n",
 281                         pci_name(rdev->lldi.pdev), wr_waitp->ret, hwtid, qpid);
 282out:
 283        return wr_waitp->ret;
 284}
 285
 286int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb);
 287
 288static inline int c4iw_ref_send_wait(struct c4iw_rdev *rdev,
 289                                     struct sk_buff *skb,
 290                                     struct c4iw_wr_wait *wr_waitp,
 291                                     u32 hwtid, u32 qpid,
 292                                     const char *func)
 293{
 294        int ret;
 295
 296        pr_debug("%s wr_wait %p hwtid %u qpid %u\n", func, wr_waitp, hwtid,
 297                 qpid);
 298        c4iw_get_wr_wait(wr_waitp);
 299        ret = c4iw_ofld_send(rdev, skb);
 300        if (ret) {
 301                c4iw_put_wr_wait(wr_waitp);
 302                return ret;
 303        }
 304        return c4iw_wait_for_reply(rdev, wr_waitp, hwtid, qpid, func);
 305}
 306
 307enum db_state {
 308        NORMAL = 0,
 309        FLOW_CONTROL = 1,
 310        RECOVERY = 2,
 311        STOPPED = 3
 312};
 313
 314struct c4iw_dev {
 315        struct ib_device ibdev;
 316        struct c4iw_rdev rdev;
 317        u32 device_cap_flags;
 318        struct xarray cqs;
 319        struct xarray qps;
 320        struct xarray mrs;
 321        struct mutex db_mutex;
 322        struct dentry *debugfs_root;
 323        enum db_state db_state;
 324        struct xarray hwtids;
 325        struct xarray atids;
 326        struct xarray stids;
 327        struct list_head db_fc_list;
 328        u32 avail_ird;
 329        wait_queue_head_t wait;
 330};
 331
 332struct uld_ctx {
 333        struct list_head entry;
 334        struct cxgb4_lld_info lldi;
 335        struct c4iw_dev *dev;
 336        struct work_struct reg_work;
 337};
 338
 339static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
 340{
 341        return container_of(ibdev, struct c4iw_dev, ibdev);
 342}
 343
 344static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)
 345{
 346        return xa_load(&rhp->cqs, cqid);
 347}
 348
 349static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)
 350{
 351        return xa_load(&rhp->qps, qpid);
 352}
 353
 354extern uint c4iw_max_read_depth;
 355
 356static inline int cur_max_read_depth(struct c4iw_dev *dev)
 357{
 358        return min(dev->rdev.lldi.max_ordird_qp, c4iw_max_read_depth);
 359}
 360
 361struct c4iw_pd {
 362        struct ib_pd ibpd;
 363        u32 pdid;
 364        struct c4iw_dev *rhp;
 365};
 366
 367static inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd)
 368{
 369        return container_of(ibpd, struct c4iw_pd, ibpd);
 370}
 371
 372struct tpt_attributes {
 373        u64 len;
 374        u64 va_fbo;
 375        enum fw_ri_mem_perms perms;
 376        u32 stag;
 377        u32 pdid;
 378        u32 qpid;
 379        u32 pbl_addr;
 380        u32 pbl_size;
 381        u32 state:1;
 382        u32 type:2;
 383        u32 rsvd:1;
 384        u32 remote_invaliate_disable:1;
 385        u32 zbva:1;
 386        u32 mw_bind_enable:1;
 387        u32 page_size:5;
 388};
 389
 390struct c4iw_mr {
 391        struct ib_mr ibmr;
 392        struct ib_umem *umem;
 393        struct c4iw_dev *rhp;
 394        struct sk_buff *dereg_skb;
 395        u64 kva;
 396        struct tpt_attributes attr;
 397        u64 *mpl;
 398        dma_addr_t mpl_addr;
 399        u32 max_mpl_len;
 400        u32 mpl_len;
 401        struct c4iw_wr_wait *wr_waitp;
 402};
 403
 404static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)
 405{
 406        return container_of(ibmr, struct c4iw_mr, ibmr);
 407}
 408
 409struct c4iw_mw {
 410        struct ib_mw ibmw;
 411        struct c4iw_dev *rhp;
 412        struct sk_buff *dereg_skb;
 413        u64 kva;
 414        struct tpt_attributes attr;
 415        struct c4iw_wr_wait *wr_waitp;
 416};
 417
 418static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)
 419{
 420        return container_of(ibmw, struct c4iw_mw, ibmw);
 421}
 422
 423struct c4iw_cq {
 424        struct ib_cq ibcq;
 425        struct c4iw_dev *rhp;
 426        struct sk_buff *destroy_skb;
 427        struct t4_cq cq;
 428        spinlock_t lock;
 429        spinlock_t comp_handler_lock;
 430        refcount_t refcnt;
 431        struct completion cq_rel_comp;
 432        struct c4iw_wr_wait *wr_waitp;
 433};
 434
 435static inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq)
 436{
 437        return container_of(ibcq, struct c4iw_cq, ibcq);
 438}
 439
 440struct c4iw_mpa_attributes {
 441        u8 initiator;
 442        u8 recv_marker_enabled;
 443        u8 xmit_marker_enabled;
 444        u8 crc_enabled;
 445        u8 enhanced_rdma_conn;
 446        u8 version;
 447        u8 p2p_type;
 448};
 449
 450struct c4iw_qp_attributes {
 451        u32 scq;
 452        u32 rcq;
 453        u32 sq_num_entries;
 454        u32 rq_num_entries;
 455        u32 sq_max_sges;
 456        u32 sq_max_sges_rdma_write;
 457        u32 rq_max_sges;
 458        u32 state;
 459        u8 enable_rdma_read;
 460        u8 enable_rdma_write;
 461        u8 enable_bind;
 462        u8 enable_mmid0_fastreg;
 463        u32 max_ord;
 464        u32 max_ird;
 465        u32 pd;
 466        u32 next_state;
 467        char terminate_buffer[52];
 468        u32 terminate_msg_len;
 469        u8 is_terminate_local;
 470        struct c4iw_mpa_attributes mpa_attr;
 471        struct c4iw_ep *llp_stream_handle;
 472        u8 layer_etype;
 473        u8 ecode;
 474        u16 sq_db_inc;
 475        u16 rq_db_inc;
 476        u8 send_term;
 477};
 478
 479struct c4iw_qp {
 480        struct ib_qp ibqp;
 481        struct list_head db_fc_entry;
 482        struct c4iw_dev *rhp;
 483        struct c4iw_ep *ep;
 484        struct c4iw_qp_attributes attr;
 485        struct t4_wq wq;
 486        spinlock_t lock;
 487        struct mutex mutex;
 488        wait_queue_head_t wait;
 489        int sq_sig_all;
 490        struct c4iw_srq *srq;
 491        struct c4iw_ucontext *ucontext;
 492        struct c4iw_wr_wait *wr_waitp;
 493        struct completion qp_rel_comp;
 494        refcount_t qp_refcnt;
 495};
 496
 497static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
 498{
 499        return container_of(ibqp, struct c4iw_qp, ibqp);
 500}
 501
 502struct c4iw_srq {
 503        struct ib_srq ibsrq;
 504        struct list_head db_fc_entry;
 505        struct c4iw_dev *rhp;
 506        struct t4_srq wq;
 507        struct sk_buff *destroy_skb;
 508        u32 srq_limit;
 509        u32 pdid;
 510        int idx;
 511        u32 flags;
 512        spinlock_t lock; /* protects srq */
 513        struct c4iw_wr_wait *wr_waitp;
 514        bool armed;
 515};
 516
 517static inline struct c4iw_srq *to_c4iw_srq(struct ib_srq *ibsrq)
 518{
 519        return container_of(ibsrq, struct c4iw_srq, ibsrq);
 520}
 521
 522struct c4iw_ucontext {
 523        struct ib_ucontext ibucontext;
 524        struct c4iw_dev_ucontext uctx;
 525        u32 key;
 526        spinlock_t mmap_lock;
 527        struct list_head mmaps;
 528        bool is_32b_cqe;
 529};
 530
 531static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
 532{
 533        return container_of(c, struct c4iw_ucontext, ibucontext);
 534}
 535
 536struct c4iw_mm_entry {
 537        struct list_head entry;
 538        u64 addr;
 539        u32 key;
 540        unsigned len;
 541};
 542
 543static inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext,
 544                                                u32 key, unsigned len)
 545{
 546        struct list_head *pos, *nxt;
 547        struct c4iw_mm_entry *mm;
 548
 549        spin_lock(&ucontext->mmap_lock);
 550        list_for_each_safe(pos, nxt, &ucontext->mmaps) {
 551
 552                mm = list_entry(pos, struct c4iw_mm_entry, entry);
 553                if (mm->key == key && mm->len == len) {
 554                        list_del_init(&mm->entry);
 555                        spin_unlock(&ucontext->mmap_lock);
 556                        pr_debug("key 0x%x addr 0x%llx len %d\n", key,
 557                                 (unsigned long long)mm->addr, mm->len);
 558                        return mm;
 559                }
 560        }
 561        spin_unlock(&ucontext->mmap_lock);
 562        return NULL;
 563}
 564
 565static inline void insert_mmap(struct c4iw_ucontext *ucontext,
 566                               struct c4iw_mm_entry *mm)
 567{
 568        spin_lock(&ucontext->mmap_lock);
 569        pr_debug("key 0x%x addr 0x%llx len %d\n",
 570                 mm->key, (unsigned long long)mm->addr, mm->len);
 571        list_add_tail(&mm->entry, &ucontext->mmaps);
 572        spin_unlock(&ucontext->mmap_lock);
 573}
 574
 575enum c4iw_qp_attr_mask {
 576        C4IW_QP_ATTR_NEXT_STATE = 1 << 0,
 577        C4IW_QP_ATTR_SQ_DB = 1<<1,
 578        C4IW_QP_ATTR_RQ_DB = 1<<2,
 579        C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7,
 580        C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8,
 581        C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9,
 582        C4IW_QP_ATTR_MAX_ORD = 1 << 11,
 583        C4IW_QP_ATTR_MAX_IRD = 1 << 12,
 584        C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22,
 585        C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23,
 586        C4IW_QP_ATTR_MPA_ATTR = 1 << 24,
 587        C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25,
 588        C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ |
 589                                     C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
 590                                     C4IW_QP_ATTR_MAX_ORD |
 591                                     C4IW_QP_ATTR_MAX_IRD |
 592                                     C4IW_QP_ATTR_LLP_STREAM_HANDLE |
 593                                     C4IW_QP_ATTR_STREAM_MSG_BUFFER |
 594                                     C4IW_QP_ATTR_MPA_ATTR |
 595                                     C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE)
 596};
 597
 598int c4iw_modify_qp(struct c4iw_dev *rhp,
 599                                struct c4iw_qp *qhp,
 600                                enum c4iw_qp_attr_mask mask,
 601                                struct c4iw_qp_attributes *attrs,
 602                                int internal);
 603
 604enum c4iw_qp_state {
 605        C4IW_QP_STATE_IDLE,
 606        C4IW_QP_STATE_RTS,
 607        C4IW_QP_STATE_ERROR,
 608        C4IW_QP_STATE_TERMINATE,
 609        C4IW_QP_STATE_CLOSING,
 610        C4IW_QP_STATE_TOT
 611};
 612
 613static inline int c4iw_convert_state(enum ib_qp_state ib_state)
 614{
 615        switch (ib_state) {
 616        case IB_QPS_RESET:
 617        case IB_QPS_INIT:
 618                return C4IW_QP_STATE_IDLE;
 619        case IB_QPS_RTS:
 620                return C4IW_QP_STATE_RTS;
 621        case IB_QPS_SQD:
 622                return C4IW_QP_STATE_CLOSING;
 623        case IB_QPS_SQE:
 624                return C4IW_QP_STATE_TERMINATE;
 625        case IB_QPS_ERR:
 626                return C4IW_QP_STATE_ERROR;
 627        default:
 628                return -1;
 629        }
 630}
 631
 632static inline int to_ib_qp_state(int c4iw_qp_state)
 633{
 634        switch (c4iw_qp_state) {
 635        case C4IW_QP_STATE_IDLE:
 636                return IB_QPS_INIT;
 637        case C4IW_QP_STATE_RTS:
 638                return IB_QPS_RTS;
 639        case C4IW_QP_STATE_CLOSING:
 640                return IB_QPS_SQD;
 641        case C4IW_QP_STATE_TERMINATE:
 642                return IB_QPS_SQE;
 643        case C4IW_QP_STATE_ERROR:
 644                return IB_QPS_ERR;
 645        }
 646        return IB_QPS_ERR;
 647}
 648
 649static inline u32 c4iw_ib_to_tpt_access(int a)
 650{
 651        return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) |
 652               (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) |
 653               (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) |
 654               FW_RI_MEM_ACCESS_LOCAL_READ;
 655}
 656
 657enum c4iw_mmid_state {
 658        C4IW_STAG_STATE_VALID,
 659        C4IW_STAG_STATE_INVALID
 660};
 661
 662#define C4IW_NODE_DESC "cxgb4 Chelsio Communications"
 663
 664#define MPA_KEY_REQ "MPA ID Req Frame"
 665#define MPA_KEY_REP "MPA ID Rep Frame"
 666
 667#define MPA_MAX_PRIVATE_DATA    256
 668#define MPA_ENHANCED_RDMA_CONN  0x10
 669#define MPA_REJECT              0x20
 670#define MPA_CRC                 0x40
 671#define MPA_MARKERS             0x80
 672#define MPA_FLAGS_MASK          0xE0
 673
 674#define MPA_V2_PEER2PEER_MODEL          0x8000
 675#define MPA_V2_ZERO_LEN_FPDU_RTR        0x4000
 676#define MPA_V2_RDMA_WRITE_RTR           0x8000
 677#define MPA_V2_RDMA_READ_RTR            0x4000
 678#define MPA_V2_IRD_ORD_MASK             0x3FFF
 679
 680#define c4iw_put_ep(ep) {                                               \
 681        pr_debug("put_ep ep %p refcnt %d\n",            \
 682                 ep, kref_read(&((ep)->kref)));                         \
 683        WARN_ON(kref_read(&((ep)->kref)) < 1);                          \
 684        kref_put(&((ep)->kref), _c4iw_free_ep);                         \
 685}
 686
 687#define c4iw_get_ep(ep) {                                               \
 688        pr_debug("get_ep ep %p, refcnt %d\n",           \
 689                 ep, kref_read(&((ep)->kref)));                         \
 690        kref_get(&((ep)->kref));                                        \
 691}
 692void _c4iw_free_ep(struct kref *kref);
 693
 694struct mpa_message {
 695        u8 key[16];
 696        u8 flags;
 697        u8 revision;
 698        __be16 private_data_size;
 699        u8 private_data[];
 700};
 701
 702struct mpa_v2_conn_params {
 703        __be16 ird;
 704        __be16 ord;
 705};
 706
 707struct terminate_message {
 708        u8 layer_etype;
 709        u8 ecode;
 710        __be16 hdrct_rsvd;
 711        u8 len_hdrs[];
 712};
 713
 714#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28)
 715
 716enum c4iw_layers_types {
 717        LAYER_RDMAP             = 0x00,
 718        LAYER_DDP               = 0x10,
 719        LAYER_MPA               = 0x20,
 720        RDMAP_LOCAL_CATA        = 0x00,
 721        RDMAP_REMOTE_PROT       = 0x01,
 722        RDMAP_REMOTE_OP         = 0x02,
 723        DDP_LOCAL_CATA          = 0x00,
 724        DDP_TAGGED_ERR          = 0x01,
 725        DDP_UNTAGGED_ERR        = 0x02,
 726        DDP_LLP                 = 0x03
 727};
 728
 729enum c4iw_rdma_ecodes {
 730        RDMAP_INV_STAG          = 0x00,
 731        RDMAP_BASE_BOUNDS       = 0x01,
 732        RDMAP_ACC_VIOL          = 0x02,
 733        RDMAP_STAG_NOT_ASSOC    = 0x03,
 734        RDMAP_TO_WRAP           = 0x04,
 735        RDMAP_INV_VERS          = 0x05,
 736        RDMAP_INV_OPCODE        = 0x06,
 737        RDMAP_STREAM_CATA       = 0x07,
 738        RDMAP_GLOBAL_CATA       = 0x08,
 739        RDMAP_CANT_INV_STAG     = 0x09,
 740        RDMAP_UNSPECIFIED       = 0xff
 741};
 742
 743enum c4iw_ddp_ecodes {
 744        DDPT_INV_STAG           = 0x00,
 745        DDPT_BASE_BOUNDS        = 0x01,
 746        DDPT_STAG_NOT_ASSOC     = 0x02,
 747        DDPT_TO_WRAP            = 0x03,
 748        DDPT_INV_VERS           = 0x04,
 749        DDPU_INV_QN             = 0x01,
 750        DDPU_INV_MSN_NOBUF      = 0x02,
 751        DDPU_INV_MSN_RANGE      = 0x03,
 752        DDPU_INV_MO             = 0x04,
 753        DDPU_MSG_TOOBIG         = 0x05,
 754        DDPU_INV_VERS           = 0x06
 755};
 756
 757enum c4iw_mpa_ecodes {
 758        MPA_CRC_ERR             = 0x02,
 759        MPA_MARKER_ERR          = 0x03,
 760        MPA_LOCAL_CATA          = 0x05,
 761        MPA_INSUFF_IRD          = 0x06,
 762        MPA_NOMATCH_RTR         = 0x07,
 763};
 764
 765enum c4iw_ep_state {
 766        IDLE = 0,
 767        LISTEN,
 768        CONNECTING,
 769        MPA_REQ_WAIT,
 770        MPA_REQ_SENT,
 771        MPA_REQ_RCVD,
 772        MPA_REP_SENT,
 773        FPDU_MODE,
 774        ABORTING,
 775        CLOSING,
 776        MORIBUND,
 777        DEAD,
 778};
 779
 780enum c4iw_ep_flags {
 781        PEER_ABORT_IN_PROGRESS  = 0,
 782        ABORT_REQ_IN_PROGRESS   = 1,
 783        RELEASE_RESOURCES       = 2,
 784        CLOSE_SENT              = 3,
 785        TIMEOUT                 = 4,
 786        QP_REFERENCED           = 5,
 787        STOP_MPA_TIMER          = 7,
 788};
 789
 790enum c4iw_ep_history {
 791        ACT_OPEN_REQ            = 0,
 792        ACT_OFLD_CONN           = 1,
 793        ACT_OPEN_RPL            = 2,
 794        ACT_ESTAB               = 3,
 795        PASS_ACCEPT_REQ         = 4,
 796        PASS_ESTAB              = 5,
 797        ABORT_UPCALL            = 6,
 798        ESTAB_UPCALL            = 7,
 799        CLOSE_UPCALL            = 8,
 800        ULP_ACCEPT              = 9,
 801        ULP_REJECT              = 10,
 802        TIMEDOUT                = 11,
 803        PEER_ABORT              = 12,
 804        PEER_CLOSE              = 13,
 805        CONNREQ_UPCALL          = 14,
 806        ABORT_CONN              = 15,
 807        DISCONN_UPCALL          = 16,
 808        EP_DISC_CLOSE           = 17,
 809        EP_DISC_ABORT           = 18,
 810        CONN_RPL_UPCALL         = 19,
 811        ACT_RETRY_NOMEM         = 20,
 812        ACT_RETRY_INUSE         = 21,
 813        CLOSE_CON_RPL           = 22,
 814        EP_DISC_FAIL            = 24,
 815        QP_REFED                = 25,
 816        QP_DEREFED              = 26,
 817        CM_ID_REFED             = 27,
 818        CM_ID_DEREFED           = 28,
 819};
 820
 821enum conn_pre_alloc_buffers {
 822        CN_ABORT_REQ_BUF,
 823        CN_ABORT_RPL_BUF,
 824        CN_CLOSE_CON_REQ_BUF,
 825        CN_DESTROY_BUF,
 826        CN_FLOWC_BUF,
 827        CN_MAX_CON_BUF
 828};
 829
 830enum {
 831        FLOWC_LEN = offsetof(struct fw_flowc_wr, mnemval[FW_FLOWC_MNEM_MAX])
 832};
 833
 834union cpl_wr_size {
 835        struct cpl_abort_req abrt_req;
 836        struct cpl_abort_rpl abrt_rpl;
 837        struct fw_ri_wr ri_req;
 838        struct cpl_close_con_req close_req;
 839        char flowc_buf[FLOWC_LEN];
 840};
 841
 842struct c4iw_ep_common {
 843        struct iw_cm_id *cm_id;
 844        struct c4iw_qp *qp;
 845        struct c4iw_dev *dev;
 846        struct sk_buff_head ep_skb_list;
 847        enum c4iw_ep_state state;
 848        struct kref kref;
 849        struct mutex mutex;
 850        struct sockaddr_storage local_addr;
 851        struct sockaddr_storage remote_addr;
 852        struct c4iw_wr_wait *wr_waitp;
 853        unsigned long flags;
 854        unsigned long history;
 855};
 856
 857struct c4iw_listen_ep {
 858        struct c4iw_ep_common com;
 859        unsigned int stid;
 860        int backlog;
 861};
 862
 863struct c4iw_ep_stats {
 864        unsigned connect_neg_adv;
 865        unsigned abort_neg_adv;
 866};
 867
 868struct c4iw_ep {
 869        struct c4iw_ep_common com;
 870        struct c4iw_ep *parent_ep;
 871        struct timer_list timer;
 872        struct list_head entry;
 873        unsigned int atid;
 874        u32 hwtid;
 875        u32 snd_seq;
 876        u32 rcv_seq;
 877        struct l2t_entry *l2t;
 878        struct dst_entry *dst;
 879        struct sk_buff *mpa_skb;
 880        struct c4iw_mpa_attributes mpa_attr;
 881        u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA];
 882        unsigned int mpa_pkt_len;
 883        u32 ird;
 884        u32 ord;
 885        u32 smac_idx;
 886        u32 tx_chan;
 887        u32 mtu;
 888        u16 mss;
 889        u16 emss;
 890        u16 plen;
 891        u16 rss_qid;
 892        u16 txq_idx;
 893        u16 ctrlq_idx;
 894        u8 tos;
 895        u8 retry_with_mpa_v1;
 896        u8 tried_with_mpa_v1;
 897        unsigned int retry_count;
 898        int snd_win;
 899        int rcv_win;
 900        u32 snd_wscale;
 901        struct c4iw_ep_stats stats;
 902        u32 srqe_idx;
 903        u32 rx_pdu_out_cnt;
 904        struct sk_buff *peer_abort_skb;
 905};
 906
 907static inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id)
 908{
 909        return cm_id->provider_data;
 910}
 911
 912static inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id)
 913{
 914        return cm_id->provider_data;
 915}
 916
 917static inline int ocqp_supported(const struct cxgb4_lld_info *infop)
 918{
 919#if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64)
 920        return infop->vr->ocq.size > 0;
 921#else
 922        return 0;
 923#endif
 924}
 925
 926u32 c4iw_id_alloc(struct c4iw_id_table *alloc);
 927void c4iw_id_free(struct c4iw_id_table *alloc, u32 obj);
 928int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num,
 929                        u32 reserved, u32 flags);
 930void c4iw_id_table_free(struct c4iw_id_table *alloc);
 931
 932typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct sk_buff *skb);
 933
 934int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
 935                     struct l2t_entry *l2t);
 936void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,
 937                   struct c4iw_dev_ucontext *uctx);
 938u32 c4iw_get_resource(struct c4iw_id_table *id_table);
 939void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);
 940int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt,
 941                       u32 nr_pdid, u32 nr_srqt);
 942int c4iw_init_ctrl_qp(struct c4iw_rdev *rdev);
 943int c4iw_pblpool_create(struct c4iw_rdev *rdev);
 944int c4iw_rqtpool_create(struct c4iw_rdev *rdev);
 945int c4iw_ocqp_pool_create(struct c4iw_rdev *rdev);
 946void c4iw_pblpool_destroy(struct c4iw_rdev *rdev);
 947void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev);
 948void c4iw_ocqp_pool_destroy(struct c4iw_rdev *rdev);
 949void c4iw_destroy_resource(struct c4iw_resource *rscp);
 950int c4iw_destroy_ctrl_qp(struct c4iw_rdev *rdev);
 951void c4iw_register_device(struct work_struct *work);
 952void c4iw_unregister_device(struct c4iw_dev *dev);
 953int __init c4iw_cm_init(void);
 954void c4iw_cm_term(void);
 955void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
 956                               struct c4iw_dev_ucontext *uctx);
 957void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,
 958                            struct c4iw_dev_ucontext *uctx);
 959int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 960int c4iw_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 961                   const struct ib_send_wr **bad_wr);
 962int c4iw_post_receive(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
 963                      const struct ib_recv_wr **bad_wr);
 964int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
 965int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);
 966int c4iw_destroy_listen(struct iw_cm_id *cm_id);
 967int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
 968int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
 969void c4iw_qp_add_ref(struct ib_qp *qp);
 970void c4iw_qp_rem_ref(struct ib_qp *qp);
 971struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
 972                            u32 max_num_sg);
 973int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 974                   unsigned int *sg_offset);
 975void c4iw_dealloc(struct uld_ctx *ctx);
 976struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
 977                                           u64 length, u64 virt, int acc,
 978                                           struct ib_udata *udata);
 979struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
 980int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
 981int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
 982void c4iw_cq_rem_ref(struct c4iw_cq *chp);
 983int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 984                   struct ib_udata *udata);
 985int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 986int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr,
 987                    enum ib_srq_attr_mask srq_attr_mask,
 988                    struct ib_udata *udata);
 989int c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);
 990int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs,
 991                    struct ib_udata *udata);
 992int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
 993struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
 994                             struct ib_qp_init_attr *attrs,
 995                             struct ib_udata *udata);
 996int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 997                                 int attr_mask, struct ib_udata *udata);
 998int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 999                     int attr_mask, struct ib_qp_init_attr *init_attr);
1000struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn);
1001u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size);
1002void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
1003u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);
1004void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
1005u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size);
1006void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size);
1007void c4iw_flush_hw_cq(struct c4iw_cq *chp, struct c4iw_qp *flush_qhp);
1008void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
1009int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
1010int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
1011int c4iw_flush_sq(struct c4iw_qp *qhp);
1012int c4iw_ev_handler(struct c4iw_dev *rnicp, u32 qid);
1013u16 c4iw_rqes_posted(struct c4iw_qp *qhp);
1014int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);
1015u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
1016void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
1017                struct c4iw_dev_ucontext *uctx);
1018u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
1019void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
1020                struct c4iw_dev_ucontext *uctx);
1021void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
1022
1023extern struct cxgb4_client t4c_client;
1024extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
1025void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
1026                              enum cxgb4_bar2_qtype qtype,
1027                              unsigned int *pbar2_qid, u64 *pbar2_pa);
1028int c4iw_alloc_srq_idx(struct c4iw_rdev *rdev);
1029void c4iw_free_srq_idx(struct c4iw_rdev *rdev, int idx);
1030extern void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe);
1031extern int c4iw_wr_log;
1032extern int db_fc_threshold;
1033extern int db_coalescing_threshold;
1034extern int use_dsgl;
1035void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);
1036void c4iw_dispatch_srq_limit_reached_event(struct c4iw_srq *srq);
1037void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16);
1038void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx);
1039int c4iw_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
1040                       const struct ib_recv_wr **bad_wr);
1041struct c4iw_wr_wait *c4iw_alloc_wr_wait(gfp_t gfp);
1042
1043int c4iw_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr);
1044int c4iw_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);
1045int c4iw_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);
1046int c4iw_fill_res_cm_id_entry(struct sk_buff *msg, struct rdma_cm_id *cm_id);
1047
1048#endif
1049