linux/drivers/infiniband/hw/amso1100/c2_qp.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
   4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   6 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
   7 *
   8 * This software is available to you under a choice of one of two
   9 * licenses.  You may choose to be licensed under the terms of the GNU
  10 * General Public License (GPL) Version 2, available from the file
  11 * COPYING in the main directory of this source tree, or the
  12 * OpenIB.org BSD license below:
  13 *
  14 *     Redistribution and use in source and binary forms, with or
  15 *     without modification, are permitted provided that the following
  16 *     conditions are met:
  17 *
  18 *      - Redistributions of source code must retain the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer.
  21 *
  22 *      - Redistributions in binary form must reproduce the above
  23 *        copyright notice, this list of conditions and the following
  24 *        disclaimer in the documentation and/or other materials
  25 *        provided with the distribution.
  26 *
  27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34 * SOFTWARE.
  35 *
  36 */
  37
  38#include <linux/delay.h>
  39#include <linux/gfp.h>
  40
  41#include "c2.h"
  42#include "c2_vq.h"
  43#include "c2_status.h"
  44
  45#define C2_MAX_ORD_PER_QP 128
  46#define C2_MAX_IRD_PER_QP 128
  47
  48#define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
  49#define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
  50#define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
  51
  52#define NO_SUPPORT -1
  53static const u8 c2_opcode[] = {
  54        [IB_WR_SEND] = C2_WR_TYPE_SEND,
  55        [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
  56        [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
  57        [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
  58        [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
  59        [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
  60        [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
  61};
  62
  63static int to_c2_state(enum ib_qp_state ib_state)
  64{
  65        switch (ib_state) {
  66        case IB_QPS_RESET:
  67                return C2_QP_STATE_IDLE;
  68        case IB_QPS_RTS:
  69                return C2_QP_STATE_RTS;
  70        case IB_QPS_SQD:
  71                return C2_QP_STATE_CLOSING;
  72        case IB_QPS_SQE:
  73                return C2_QP_STATE_CLOSING;
  74        case IB_QPS_ERR:
  75                return C2_QP_STATE_ERROR;
  76        default:
  77                return -1;
  78        }
  79}
  80
  81static int to_ib_state(enum c2_qp_state c2_state)
  82{
  83        switch (c2_state) {
  84        case C2_QP_STATE_IDLE:
  85                return IB_QPS_RESET;
  86        case C2_QP_STATE_CONNECTING:
  87                return IB_QPS_RTR;
  88        case C2_QP_STATE_RTS:
  89                return IB_QPS_RTS;
  90        case C2_QP_STATE_CLOSING:
  91                return IB_QPS_SQD;
  92        case C2_QP_STATE_ERROR:
  93                return IB_QPS_ERR;
  94        case C2_QP_STATE_TERMINATE:
  95                return IB_QPS_SQE;
  96        default:
  97                return -1;
  98        }
  99}
 100
 101static const char *to_ib_state_str(int ib_state)
 102{
 103        static const char *state_str[] = {
 104                "IB_QPS_RESET",
 105                "IB_QPS_INIT",
 106                "IB_QPS_RTR",
 107                "IB_QPS_RTS",
 108                "IB_QPS_SQD",
 109                "IB_QPS_SQE",
 110                "IB_QPS_ERR"
 111        };
 112        if (ib_state < IB_QPS_RESET ||
 113            ib_state > IB_QPS_ERR)
 114                return "<invalid IB QP state>";
 115
 116        ib_state -= IB_QPS_RESET;
 117        return state_str[ib_state];
 118}
 119
 120void c2_set_qp_state(struct c2_qp *qp, int c2_state)
 121{
 122        int new_state = to_ib_state(c2_state);
 123
 124        pr_debug("%s: qp[%p] state modify %s --> %s\n",
 125               __func__,
 126                qp,
 127                to_ib_state_str(qp->state),
 128                to_ib_state_str(new_state));
 129        qp->state = new_state;
 130}
 131
 132#define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
 133
 134int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
 135                 struct ib_qp_attr *attr, int attr_mask)
 136{
 137        struct c2wr_qp_modify_req wr;
 138        struct c2wr_qp_modify_rep *reply;
 139        struct c2_vq_req *vq_req;
 140        unsigned long flags;
 141        u8 next_state;
 142        int err;
 143
 144        pr_debug("%s:%d qp=%p, %s --> %s\n",
 145                __func__, __LINE__,
 146                qp,
 147                to_ib_state_str(qp->state),
 148                to_ib_state_str(attr->qp_state));
 149
 150        vq_req = vq_req_alloc(c2dev);
 151        if (!vq_req)
 152                return -ENOMEM;
 153
 154        c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 155        wr.hdr.context = (unsigned long) vq_req;
 156        wr.rnic_handle = c2dev->adapter_handle;
 157        wr.qp_handle = qp->adapter_handle;
 158        wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 159        wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 160        wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 161        wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 162
 163        if (attr_mask & IB_QP_STATE) {
 164                /* Ensure the state is valid */
 165                if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) {
 166                        err = -EINVAL;
 167                        goto bail0;
 168                }
 169
 170                wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
 171
 172                if (attr->qp_state == IB_QPS_ERR) {
 173                        spin_lock_irqsave(&qp->lock, flags);
 174                        if (qp->cm_id && qp->state == IB_QPS_RTS) {
 175                                pr_debug("Generating CLOSE event for QP-->ERR, "
 176                                        "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 177                                /* Generate an CLOSE event */
 178                                vq_req->cm_id = qp->cm_id;
 179                                vq_req->event = IW_CM_EVENT_CLOSE;
 180                        }
 181                        spin_unlock_irqrestore(&qp->lock, flags);
 182                }
 183                next_state =  attr->qp_state;
 184
 185        } else if (attr_mask & IB_QP_CUR_STATE) {
 186
 187                if (attr->cur_qp_state != IB_QPS_RTR &&
 188                    attr->cur_qp_state != IB_QPS_RTS &&
 189                    attr->cur_qp_state != IB_QPS_SQD &&
 190                    attr->cur_qp_state != IB_QPS_SQE) {
 191                        err = -EINVAL;
 192                        goto bail0;
 193                } else
 194                        wr.next_qp_state =
 195                            cpu_to_be32(to_c2_state(attr->cur_qp_state));
 196
 197                next_state = attr->cur_qp_state;
 198
 199        } else {
 200                err = 0;
 201                goto bail0;
 202        }
 203
 204        /* reference the request struct */
 205        vq_req_get(c2dev, vq_req);
 206
 207        err = vq_send_wr(c2dev, (union c2wr *) & wr);
 208        if (err) {
 209                vq_req_put(c2dev, vq_req);
 210                goto bail0;
 211        }
 212
 213        err = vq_wait_for_reply(c2dev, vq_req);
 214        if (err)
 215                goto bail0;
 216
 217        reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
 218        if (!reply) {
 219                err = -ENOMEM;
 220                goto bail0;
 221        }
 222
 223        err = c2_errno(reply);
 224        if (!err)
 225                qp->state = next_state;
 226#ifdef DEBUG
 227        else
 228                pr_debug("%s: c2_errno=%d\n", __func__, err);
 229#endif
 230        /*
 231         * If we're going to error and generating the event here, then
 232         * we need to remove the reference because there will be no
 233         * close event generated by the adapter
 234        */
 235        spin_lock_irqsave(&qp->lock, flags);
 236        if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
 237                qp->cm_id->rem_ref(qp->cm_id);
 238                qp->cm_id = NULL;
 239        }
 240        spin_unlock_irqrestore(&qp->lock, flags);
 241
 242        vq_repbuf_free(c2dev, reply);
 243      bail0:
 244        vq_req_free(c2dev, vq_req);
 245
 246        pr_debug("%s:%d qp=%p, cur_state=%s\n",
 247                __func__, __LINE__,
 248                qp,
 249                to_ib_state_str(qp->state));
 250        return err;
 251}
 252
 253int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
 254                          int ord, int ird)
 255{
 256        struct c2wr_qp_modify_req wr;
 257        struct c2wr_qp_modify_rep *reply;
 258        struct c2_vq_req *vq_req;
 259        int err;
 260
 261        vq_req = vq_req_alloc(c2dev);
 262        if (!vq_req)
 263                return -ENOMEM;
 264
 265        c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 266        wr.hdr.context = (unsigned long) vq_req;
 267        wr.rnic_handle = c2dev->adapter_handle;
 268        wr.qp_handle = qp->adapter_handle;
 269        wr.ord = cpu_to_be32(ord);
 270        wr.ird = cpu_to_be32(ird);
 271        wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 272        wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 273        wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 274
 275        /* reference the request struct */
 276        vq_req_get(c2dev, vq_req);
 277
 278        err = vq_send_wr(c2dev, (union c2wr *) & wr);
 279        if (err) {
 280                vq_req_put(c2dev, vq_req);
 281                goto bail0;
 282        }
 283
 284        err = vq_wait_for_reply(c2dev, vq_req);
 285        if (err)
 286                goto bail0;
 287
 288        reply = (struct c2wr_qp_modify_rep *) (unsigned long)
 289                vq_req->reply_msg;
 290        if (!reply) {
 291                err = -ENOMEM;
 292                goto bail0;
 293        }
 294
 295        err = c2_errno(reply);
 296        vq_repbuf_free(c2dev, reply);
 297      bail0:
 298        vq_req_free(c2dev, vq_req);
 299        return err;
 300}
 301
 302static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 303{
 304        struct c2_vq_req *vq_req;
 305        struct c2wr_qp_destroy_req wr;
 306        struct c2wr_qp_destroy_rep *reply;
 307        unsigned long flags;
 308        int err;
 309
 310        /*
 311         * Allocate a verb request message
 312         */
 313        vq_req = vq_req_alloc(c2dev);
 314        if (!vq_req) {
 315                return -ENOMEM;
 316        }
 317
 318        /*
 319         * Initialize the WR
 320         */
 321        c2_wr_set_id(&wr, CCWR_QP_DESTROY);
 322        wr.hdr.context = (unsigned long) vq_req;
 323        wr.rnic_handle = c2dev->adapter_handle;
 324        wr.qp_handle = qp->adapter_handle;
 325
 326        /*
 327         * reference the request struct.  dereferenced in the int handler.
 328         */
 329        vq_req_get(c2dev, vq_req);
 330
 331        spin_lock_irqsave(&qp->lock, flags);
 332        if (qp->cm_id && qp->state == IB_QPS_RTS) {
 333                pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
 334                        "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 335                /* Generate an CLOSE event */
 336                vq_req->qp = qp;
 337                vq_req->cm_id = qp->cm_id;
 338                vq_req->event = IW_CM_EVENT_CLOSE;
 339        }
 340        spin_unlock_irqrestore(&qp->lock, flags);
 341
 342        /*
 343         * Send WR to adapter
 344         */
 345        err = vq_send_wr(c2dev, (union c2wr *) & wr);
 346        if (err) {
 347                vq_req_put(c2dev, vq_req);
 348                goto bail0;
 349        }
 350
 351        /*
 352         * Wait for reply from adapter
 353         */
 354        err = vq_wait_for_reply(c2dev, vq_req);
 355        if (err) {
 356                goto bail0;
 357        }
 358
 359        /*
 360         * Process reply
 361         */
 362        reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
 363        if (!reply) {
 364                err = -ENOMEM;
 365                goto bail0;
 366        }
 367
 368        spin_lock_irqsave(&qp->lock, flags);
 369        if (qp->cm_id) {
 370                qp->cm_id->rem_ref(qp->cm_id);
 371                qp->cm_id = NULL;
 372        }
 373        spin_unlock_irqrestore(&qp->lock, flags);
 374
 375        vq_repbuf_free(c2dev, reply);
 376      bail0:
 377        vq_req_free(c2dev, vq_req);
 378        return err;
 379}
 380
 381static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
 382{
 383        int ret;
 384
 385        idr_preload(GFP_KERNEL);
 386        spin_lock_irq(&c2dev->qp_table.lock);
 387
 388        ret = idr_alloc_cyclic(&c2dev->qp_table.idr, qp, 0, 0, GFP_NOWAIT);
 389        if (ret >= 0)
 390                qp->qpn = ret;
 391
 392        spin_unlock_irq(&c2dev->qp_table.lock);
 393        idr_preload_end();
 394        return ret < 0 ? ret : 0;
 395}
 396
 397static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
 398{
 399        spin_lock_irq(&c2dev->qp_table.lock);
 400        idr_remove(&c2dev->qp_table.idr, qpn);
 401        spin_unlock_irq(&c2dev->qp_table.lock);
 402}
 403
 404struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
 405{
 406        unsigned long flags;
 407        struct c2_qp *qp;
 408
 409        spin_lock_irqsave(&c2dev->qp_table.lock, flags);
 410        qp = idr_find(&c2dev->qp_table.idr, qpn);
 411        spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
 412        return qp;
 413}
 414
 415int c2_alloc_qp(struct c2_dev *c2dev,
 416                struct c2_pd *pd,
 417                struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
 418{
 419        struct c2wr_qp_create_req wr;
 420        struct c2wr_qp_create_rep *reply;
 421        struct c2_vq_req *vq_req;
 422        struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
 423        struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
 424        unsigned long peer_pa;
 425        u32 q_size, msg_size, mmap_size;
 426        void __iomem *mmap;
 427        int err;
 428
 429        err = c2_alloc_qpn(c2dev, qp);
 430        if (err)
 431                return err;
 432        qp->ibqp.qp_num = qp->qpn;
 433        qp->ibqp.qp_type = IB_QPT_RC;
 434
 435        /* Allocate the SQ and RQ shared pointers */
 436        qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 437                                         &qp->sq_mq.shared_dma, GFP_KERNEL);
 438        if (!qp->sq_mq.shared) {
 439                err = -ENOMEM;
 440                goto bail0;
 441        }
 442
 443        qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 444                                         &qp->rq_mq.shared_dma, GFP_KERNEL);
 445        if (!qp->rq_mq.shared) {
 446                err = -ENOMEM;
 447                goto bail1;
 448        }
 449
 450        /* Allocate the verbs request */
 451        vq_req = vq_req_alloc(c2dev);
 452        if (vq_req == NULL) {
 453                err = -ENOMEM;
 454                goto bail2;
 455        }
 456
 457        /* Initialize the work request */
 458        memset(&wr, 0, sizeof(wr));
 459        c2_wr_set_id(&wr, CCWR_QP_CREATE);
 460        wr.hdr.context = (unsigned long) vq_req;
 461        wr.rnic_handle = c2dev->adapter_handle;
 462        wr.sq_cq_handle = send_cq->adapter_handle;
 463        wr.rq_cq_handle = recv_cq->adapter_handle;
 464        wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
 465        wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
 466        wr.srq_handle = 0;
 467        wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
 468                               QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
 469        wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 470        wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
 471        wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 472        wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
 473        wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
 474        wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
 475        wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
 476        wr.pd_id = pd->pd_id;
 477        wr.user_context = (unsigned long) qp;
 478
 479        vq_req_get(c2dev, vq_req);
 480
 481        /* Send the WR to the adapter */
 482        err = vq_send_wr(c2dev, (union c2wr *) & wr);
 483        if (err) {
 484                vq_req_put(c2dev, vq_req);
 485                goto bail3;
 486        }
 487
 488        /* Wait for the verb reply  */
 489        err = vq_wait_for_reply(c2dev, vq_req);
 490        if (err) {
 491                goto bail3;
 492        }
 493
 494        /* Process the reply */
 495        reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
 496        if (!reply) {
 497                err = -ENOMEM;
 498                goto bail3;
 499        }
 500
 501        if ((err = c2_wr_get_result(reply)) != 0) {
 502                goto bail4;
 503        }
 504
 505        /* Fill in the kernel QP struct */
 506        atomic_set(&qp->refcount, 1);
 507        qp->adapter_handle = reply->qp_handle;
 508        qp->state = IB_QPS_RESET;
 509        qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
 510        qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
 511        qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
 512        init_waitqueue_head(&qp->wait);
 513
 514        /* Initialize the SQ MQ */
 515        q_size = be32_to_cpu(reply->sq_depth);
 516        msg_size = be32_to_cpu(reply->sq_msg_size);
 517        peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
 518        mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 519        mmap = ioremap_nocache(peer_pa, mmap_size);
 520        if (!mmap) {
 521                err = -ENOMEM;
 522                goto bail5;
 523        }
 524
 525        c2_mq_req_init(&qp->sq_mq,
 526                       be32_to_cpu(reply->sq_mq_index),
 527                       q_size,
 528                       msg_size,
 529                       mmap + sizeof(struct c2_mq_shared),      /* pool start */
 530                       mmap,                            /* peer */
 531                       C2_MQ_ADAPTER_TARGET);
 532
 533        /* Initialize the RQ mq */
 534        q_size = be32_to_cpu(reply->rq_depth);
 535        msg_size = be32_to_cpu(reply->rq_msg_size);
 536        peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
 537        mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 538        mmap = ioremap_nocache(peer_pa, mmap_size);
 539        if (!mmap) {
 540                err = -ENOMEM;
 541                goto bail6;
 542        }
 543
 544        c2_mq_req_init(&qp->rq_mq,
 545                       be32_to_cpu(reply->rq_mq_index),
 546                       q_size,
 547                       msg_size,
 548                       mmap + sizeof(struct c2_mq_shared),      /* pool start */
 549                       mmap,                            /* peer */
 550                       C2_MQ_ADAPTER_TARGET);
 551
 552        vq_repbuf_free(c2dev, reply);
 553        vq_req_free(c2dev, vq_req);
 554
 555        return 0;
 556
 557      bail6:
 558        iounmap(qp->sq_mq.peer);
 559      bail5:
 560        destroy_qp(c2dev, qp);
 561      bail4:
 562        vq_repbuf_free(c2dev, reply);
 563      bail3:
 564        vq_req_free(c2dev, vq_req);
 565      bail2:
 566        c2_free_mqsp(qp->rq_mq.shared);
 567      bail1:
 568        c2_free_mqsp(qp->sq_mq.shared);
 569      bail0:
 570        c2_free_qpn(c2dev, qp->qpn);
 571        return err;
 572}
 573
 574static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 575{
 576        if (send_cq == recv_cq)
 577                spin_lock_irq(&send_cq->lock);
 578        else if (send_cq > recv_cq) {
 579                spin_lock_irq(&send_cq->lock);
 580                spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
 581        } else {
 582                spin_lock_irq(&recv_cq->lock);
 583                spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
 584        }
 585}
 586
 587static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 588{
 589        if (send_cq == recv_cq)
 590                spin_unlock_irq(&send_cq->lock);
 591        else if (send_cq > recv_cq) {
 592                spin_unlock(&recv_cq->lock);
 593                spin_unlock_irq(&send_cq->lock);
 594        } else {
 595                spin_unlock(&send_cq->lock);
 596                spin_unlock_irq(&recv_cq->lock);
 597        }
 598}
 599
 600void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 601{
 602        struct c2_cq *send_cq;
 603        struct c2_cq *recv_cq;
 604
 605        send_cq = to_c2cq(qp->ibqp.send_cq);
 606        recv_cq = to_c2cq(qp->ibqp.recv_cq);
 607
 608        /*
 609         * Lock CQs here, so that CQ polling code can do QP lookup
 610         * without taking a lock.
 611         */
 612        c2_lock_cqs(send_cq, recv_cq);
 613        c2_free_qpn(c2dev, qp->qpn);
 614        c2_unlock_cqs(send_cq, recv_cq);
 615
 616        /*
 617         * Destroy qp in the rnic...
 618         */
 619        destroy_qp(c2dev, qp);
 620
 621        /*
 622         * Mark any unreaped CQEs as null and void.
 623         */
 624        c2_cq_clean(c2dev, qp, send_cq->cqn);
 625        if (send_cq != recv_cq)
 626                c2_cq_clean(c2dev, qp, recv_cq->cqn);
 627        /*
 628         * Unmap the MQs and return the shared pointers
 629         * to the message pool.
 630         */
 631        iounmap(qp->sq_mq.peer);
 632        iounmap(qp->rq_mq.peer);
 633        c2_free_mqsp(qp->sq_mq.shared);
 634        c2_free_mqsp(qp->rq_mq.shared);
 635
 636        atomic_dec(&qp->refcount);
 637        wait_event(qp->wait, !atomic_read(&qp->refcount));
 638}
 639
 640/*
 641 * Function: move_sgl
 642 *
 643 * Description:
 644 * Move an SGL from the user's work request struct into a CCIL Work Request
 645 * message, swapping to WR byte order and ensure the total length doesn't
 646 * overflow.
 647 *
 648 * IN:
 649 * dst          - ptr to CCIL Work Request message SGL memory.
 650 * src          - ptr to the consumers SGL memory.
 651 *
 652 * OUT: none
 653 *
 654 * Return:
 655 * CCIL status codes.
 656 */
 657static int
 658move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
 659         u8 * actual_count)
 660{
 661        u32 tot = 0;            /* running total */
 662        u8 acount = 0;          /* running total non-0 len sge's */
 663
 664        while (count > 0) {
 665                /*
 666                 * If the addition of this SGE causes the
 667                 * total SGL length to exceed 2^32-1, then
 668                 * fail-n-bail.
 669                 *
 670                 * If the current total plus the next element length
 671                 * wraps, then it will go negative and be less than the
 672                 * current total...
 673                 */
 674                if ((tot + src->length) < tot) {
 675                        return -EINVAL;
 676                }
 677                /*
 678                 * Bug: 1456 (as well as 1498 & 1643)
 679                 * Skip over any sge's supplied with len=0
 680                 */
 681                if (src->length) {
 682                        tot += src->length;
 683                        dst->stag = cpu_to_be32(src->lkey);
 684                        dst->to = cpu_to_be64(src->addr);
 685                        dst->length = cpu_to_be32(src->length);
 686                        dst++;
 687                        acount++;
 688                }
 689                src++;
 690                count--;
 691        }
 692
 693        if (acount == 0) {
 694                /*
 695                 * Bug: 1476 (as well as 1498, 1456 and 1643)
 696                 * Setup the SGL in the WR to make it easier for the RNIC.
 697                 * This way, the FW doesn't have to deal with special cases.
 698                 * Setting length=0 should be sufficient.
 699                 */
 700                dst->stag = 0;
 701                dst->to = 0;
 702                dst->length = 0;
 703        }
 704
 705        *p_len = tot;
 706        *actual_count = acount;
 707        return 0;
 708}
 709
 710/*
 711 * Function: c2_activity (private function)
 712 *
 713 * Description:
 714 * Post an mq index to the host->adapter activity fifo.
 715 *
 716 * IN:
 717 * c2dev        - ptr to c2dev structure
 718 * mq_index     - mq index to post
 719 * shared       - value most recently written to shared
 720 *
 721 * OUT:
 722 *
 723 * Return:
 724 * none
 725 */
 726static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
 727{
 728        /*
 729         * First read the register to see if the FIFO is full, and if so,
 730         * spin until it's not.  This isn't perfect -- there is no
 731         * synchronization among the clients of the register, but in
 732         * practice it prevents multiple CPU from hammering the bus
 733         * with PCI RETRY. Note that when this does happen, the card
 734         * cannot get on the bus and the card and system hang in a
 735         * deadlock -- thus the need for this code. [TOT]
 736         */
 737        while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
 738                udelay(10);
 739
 740        __raw_writel(C2_HINT_MAKE(mq_index, shared),
 741                     c2dev->regs + PCI_BAR0_ADAPTER_HINT);
 742}
 743
 744/*
 745 * Function: qp_wr_post
 746 *
 747 * Description:
 748 * This in-line function allocates a MQ msg, then moves the host-copy of
 749 * the completed WR into msg.  Then it posts the message.
 750 *
 751 * IN:
 752 * q            - ptr to user MQ.
 753 * wr           - ptr to host-copy of the WR.
 754 * qp           - ptr to user qp
 755 * size         - Number of bytes to post.  Assumed to be divisible by 4.
 756 *
 757 * OUT: none
 758 *
 759 * Return:
 760 * CCIL status codes.
 761 */
 762static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
 763{
 764        union c2wr *msg;
 765
 766        msg = c2_mq_alloc(q);
 767        if (msg == NULL) {
 768                return -EINVAL;
 769        }
 770#ifdef CCMSGMAGIC
 771        ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
 772#endif
 773
 774        /*
 775         * Since all header fields in the WR are the same as the
 776         * CQE, set the following so the adapter need not.
 777         */
 778        c2_wr_set_result(wr, CCERR_PENDING);
 779
 780        /*
 781         * Copy the wr down to the adapter
 782         */
 783        memcpy((void *) msg, (void *) wr, size);
 784
 785        c2_mq_produce(q);
 786        return 0;
 787}
 788
 789
 790int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
 791                 struct ib_send_wr **bad_wr)
 792{
 793        struct c2_dev *c2dev = to_c2dev(ibqp->device);
 794        struct c2_qp *qp = to_c2qp(ibqp);
 795        union c2wr wr;
 796        unsigned long lock_flags;
 797        int err = 0;
 798
 799        u32 flags;
 800        u32 tot_len;
 801        u8 actual_sge_count;
 802        u32 msg_size;
 803
 804        if (qp->state > IB_QPS_RTS) {
 805                err = -EINVAL;
 806                goto out;
 807        }
 808
 809        while (ib_wr) {
 810
 811                flags = 0;
 812                wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 813                if (ib_wr->send_flags & IB_SEND_SIGNALED) {
 814                        flags |= SQ_SIGNALED;
 815                }
 816
 817                switch (ib_wr->opcode) {
 818                case IB_WR_SEND:
 819                case IB_WR_SEND_WITH_INV:
 820                        if (ib_wr->opcode == IB_WR_SEND) {
 821                                if (ib_wr->send_flags & IB_SEND_SOLICITED)
 822                                        c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
 823                                else
 824                                        c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
 825                                wr.sqwr.send.remote_stag = 0;
 826                        } else {
 827                                if (ib_wr->send_flags & IB_SEND_SOLICITED)
 828                                        c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV);
 829                                else
 830                                        c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV);
 831                                wr.sqwr.send.remote_stag =
 832                                        cpu_to_be32(ib_wr->ex.invalidate_rkey);
 833                        }
 834
 835                        msg_size = sizeof(struct c2wr_send_req) +
 836                                sizeof(struct c2_data_addr) * ib_wr->num_sge;
 837                        if (ib_wr->num_sge > qp->send_sgl_depth) {
 838                                err = -EINVAL;
 839                                break;
 840                        }
 841                        if (ib_wr->send_flags & IB_SEND_FENCE) {
 842                                flags |= SQ_READ_FENCE;
 843                        }
 844                        err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
 845                                       ib_wr->sg_list,
 846                                       ib_wr->num_sge,
 847                                       &tot_len, &actual_sge_count);
 848                        wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
 849                        c2_wr_set_sge_count(&wr, actual_sge_count);
 850                        break;
 851                case IB_WR_RDMA_WRITE:
 852                        c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
 853                        msg_size = sizeof(struct c2wr_rdma_write_req) +
 854                            (sizeof(struct c2_data_addr) * ib_wr->num_sge);
 855                        if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
 856                                err = -EINVAL;
 857                                break;
 858                        }
 859                        if (ib_wr->send_flags & IB_SEND_FENCE) {
 860                                flags |= SQ_READ_FENCE;
 861                        }
 862                        wr.sqwr.rdma_write.remote_stag =
 863                            cpu_to_be32(rdma_wr(ib_wr)->rkey);
 864                        wr.sqwr.rdma_write.remote_to =
 865                            cpu_to_be64(rdma_wr(ib_wr)->remote_addr);
 866                        err = move_sgl((struct c2_data_addr *)
 867                                       & (wr.sqwr.rdma_write.data),
 868                                       ib_wr->sg_list,
 869                                       ib_wr->num_sge,
 870                                       &tot_len, &actual_sge_count);
 871                        wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
 872                        c2_wr_set_sge_count(&wr, actual_sge_count);
 873                        break;
 874                case IB_WR_RDMA_READ:
 875                        c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
 876                        msg_size = sizeof(struct c2wr_rdma_read_req);
 877
 878                        /* IWarp only suppots 1 sge for RDMA reads */
 879                        if (ib_wr->num_sge > 1) {
 880                                err = -EINVAL;
 881                                break;
 882                        }
 883
 884                        /*
 885                         * Move the local and remote stag/to/len into the WR.
 886                         */
 887                        wr.sqwr.rdma_read.local_stag =
 888                            cpu_to_be32(ib_wr->sg_list->lkey);
 889                        wr.sqwr.rdma_read.local_to =
 890                            cpu_to_be64(ib_wr->sg_list->addr);
 891                        wr.sqwr.rdma_read.remote_stag =
 892                            cpu_to_be32(rdma_wr(ib_wr)->rkey);
 893                        wr.sqwr.rdma_read.remote_to =
 894                            cpu_to_be64(rdma_wr(ib_wr)->remote_addr);
 895                        wr.sqwr.rdma_read.length =
 896                            cpu_to_be32(ib_wr->sg_list->length);
 897                        break;
 898                default:
 899                        /* error */
 900                        msg_size = 0;
 901                        err = -EINVAL;
 902                        break;
 903                }
 904
 905                /*
 906                 * If we had an error on the last wr build, then
 907                 * break out.  Possible errors include bogus WR
 908                 * type, and a bogus SGL length...
 909                 */
 910                if (err) {
 911                        break;
 912                }
 913
 914                /*
 915                 * Store flags
 916                 */
 917                c2_wr_set_flags(&wr, flags);
 918
 919                /*
 920                 * Post the puppy!
 921                 */
 922                spin_lock_irqsave(&qp->lock, lock_flags);
 923                err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
 924                if (err) {
 925                        spin_unlock_irqrestore(&qp->lock, lock_flags);
 926                        break;
 927                }
 928
 929                /*
 930                 * Enqueue mq index to activity FIFO.
 931                 */
 932                c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
 933                spin_unlock_irqrestore(&qp->lock, lock_flags);
 934
 935                ib_wr = ib_wr->next;
 936        }
 937
 938out:
 939        if (err)
 940                *bad_wr = ib_wr;
 941        return err;
 942}
 943
 944int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
 945                    struct ib_recv_wr **bad_wr)
 946{
 947        struct c2_dev *c2dev = to_c2dev(ibqp->device);
 948        struct c2_qp *qp = to_c2qp(ibqp);
 949        union c2wr wr;
 950        unsigned long lock_flags;
 951        int err = 0;
 952
 953        if (qp->state > IB_QPS_RTS) {
 954                err = -EINVAL;
 955                goto out;
 956        }
 957
 958        /*
 959         * Try and post each work request
 960         */
 961        while (ib_wr) {
 962                u32 tot_len;
 963                u8 actual_sge_count;
 964
 965                if (ib_wr->num_sge > qp->recv_sgl_depth) {
 966                        err = -EINVAL;
 967                        break;
 968                }
 969
 970                /*
 971                 * Create local host-copy of the WR
 972                 */
 973                wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 974                c2_wr_set_id(&wr, CCWR_RECV);
 975                c2_wr_set_flags(&wr, 0);
 976
 977                /* sge_count is limited to eight bits. */
 978                BUG_ON(ib_wr->num_sge >= 256);
 979                err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
 980                               ib_wr->sg_list,
 981                               ib_wr->num_sge, &tot_len, &actual_sge_count);
 982                c2_wr_set_sge_count(&wr, actual_sge_count);
 983
 984                /*
 985                 * If we had an error on the last wr build, then
 986                 * break out.  Possible errors include bogus WR
 987                 * type, and a bogus SGL length...
 988                 */
 989                if (err) {
 990                        break;
 991                }
 992
 993                spin_lock_irqsave(&qp->lock, lock_flags);
 994                err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
 995                if (err) {
 996                        spin_unlock_irqrestore(&qp->lock, lock_flags);
 997                        break;
 998                }
 999
1000                /*
1001                 * Enqueue mq index to activity FIFO
1002                 */
1003                c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
1004                spin_unlock_irqrestore(&qp->lock, lock_flags);
1005
1006                ib_wr = ib_wr->next;
1007        }
1008
1009out:
1010        if (err)
1011                *bad_wr = ib_wr;
1012        return err;
1013}
1014
1015void c2_init_qp_table(struct c2_dev *c2dev)
1016{
1017        spin_lock_init(&c2dev->qp_table.lock);
1018        idr_init(&c2dev->qp_table.idr);
1019}
1020
1021void c2_cleanup_qp_table(struct c2_dev *c2dev)
1022{
1023        idr_destroy(&c2dev->qp_table.idr);
1024}
1025