linux/drivers/net/ethernet/mellanox/mlx4/cq.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4 * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
   5 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
   6 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   7 *
   8 * This software is available to you under a choice of one of two
   9 * licenses.  You may choose to be licensed under the terms of the GNU
  10 * General Public License (GPL) Version 2, available from the file
  11 * COPYING in the main directory of this source tree, or the
  12 * OpenIB.org BSD license below:
  13 *
  14 *     Redistribution and use in source and binary forms, with or
  15 *     without modification, are permitted provided that the following
  16 *     conditions are met:
  17 *
  18 *      - Redistributions of source code must retain the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer.
  21 *
  22 *      - Redistributions in binary form must reproduce the above
  23 *        copyright notice, this list of conditions and the following
  24 *        disclaimer in the documentation and/or other materials
  25 *        provided with the distribution.
  26 *
  27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34 * SOFTWARE.
  35 */
  36
  37#include <linux/hardirq.h>
  38#include <linux/export.h>
  39
  40#include <linux/mlx4/cmd.h>
  41#include <linux/mlx4/cq.h>
  42
  43#include "mlx4.h"
  44#include "icm.h"
  45
  46#define MLX4_CQ_STATUS_OK               ( 0 << 28)
  47#define MLX4_CQ_STATUS_OVERFLOW         ( 9 << 28)
  48#define MLX4_CQ_STATUS_WRITE_FAIL       (10 << 28)
  49#define MLX4_CQ_FLAG_CC                 ( 1 << 18)
  50#define MLX4_CQ_FLAG_OI                 ( 1 << 17)
  51#define MLX4_CQ_STATE_ARMED             ( 9 <<  8)
  52#define MLX4_CQ_STATE_ARMED_SOL         ( 6 <<  8)
  53#define MLX4_EQ_STATE_FIRED             (10 <<  8)
  54
  55#define TASKLET_MAX_TIME 2
  56#define TASKLET_MAX_TIME_JIFFIES msecs_to_jiffies(TASKLET_MAX_TIME)
  57
  58void mlx4_cq_tasklet_cb(unsigned long data)
  59{
  60        unsigned long flags;
  61        unsigned long end = jiffies + TASKLET_MAX_TIME_JIFFIES;
  62        struct mlx4_eq_tasklet *ctx = (struct mlx4_eq_tasklet *)data;
  63        struct mlx4_cq *mcq, *temp;
  64
  65        spin_lock_irqsave(&ctx->lock, flags);
  66        list_splice_tail_init(&ctx->list, &ctx->process_list);
  67        spin_unlock_irqrestore(&ctx->lock, flags);
  68
  69        list_for_each_entry_safe(mcq, temp, &ctx->process_list, tasklet_ctx.list) {
  70                list_del_init(&mcq->tasklet_ctx.list);
  71                mcq->tasklet_ctx.comp(mcq);
  72                if (refcount_dec_and_test(&mcq->refcount))
  73                        complete(&mcq->free);
  74                if (time_after(jiffies, end))
  75                        break;
  76        }
  77
  78        if (!list_empty(&ctx->process_list))
  79                tasklet_schedule(&ctx->task);
  80}
  81
  82static void mlx4_add_cq_to_tasklet(struct mlx4_cq *cq)
  83{
  84        struct mlx4_eq_tasklet *tasklet_ctx = cq->tasklet_ctx.priv;
  85        unsigned long flags;
  86        bool kick;
  87
  88        spin_lock_irqsave(&tasklet_ctx->lock, flags);
  89        /* When migrating CQs between EQs will be implemented, please note
  90         * that you need to sync this point. It is possible that
  91         * while migrating a CQ, completions on the old EQs could
  92         * still arrive.
  93         */
  94        if (list_empty_careful(&cq->tasklet_ctx.list)) {
  95                refcount_inc(&cq->refcount);
  96                kick = list_empty(&tasklet_ctx->list);
  97                list_add_tail(&cq->tasklet_ctx.list, &tasklet_ctx->list);
  98                if (kick)
  99                        tasklet_schedule(&tasklet_ctx->task);
 100        }
 101        spin_unlock_irqrestore(&tasklet_ctx->lock, flags);
 102}
 103
 104void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn)
 105{
 106        struct mlx4_cq *cq;
 107
 108        rcu_read_lock();
 109        cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree,
 110                               cqn & (dev->caps.num_cqs - 1));
 111        rcu_read_unlock();
 112
 113        if (!cq) {
 114                mlx4_dbg(dev, "Completion event for bogus CQ %08x\n", cqn);
 115                return;
 116        }
 117
 118        /* Acessing the CQ outside of rcu_read_lock is safe, because
 119         * the CQ is freed only after interrupt handling is completed.
 120         */
 121        ++cq->arm_sn;
 122
 123        cq->comp(cq);
 124}
 125
 126void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type)
 127{
 128        struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
 129        struct mlx4_cq *cq;
 130
 131        rcu_read_lock();
 132        cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1));
 133        rcu_read_unlock();
 134
 135        if (!cq) {
 136                mlx4_dbg(dev, "Async event for bogus CQ %08x\n", cqn);
 137                return;
 138        }
 139
 140        /* Acessing the CQ outside of rcu_read_lock is safe, because
 141         * the CQ is freed only after interrupt handling is completed.
 142         */
 143        cq->event(cq, event_type);
 144}
 145
 146static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
 147                         int cq_num, u8 opmod)
 148{
 149        return mlx4_cmd(dev, mailbox->dma, cq_num, opmod,
 150                        MLX4_CMD_SW2HW_CQ, MLX4_CMD_TIME_CLASS_A,
 151                        MLX4_CMD_WRAPPED);
 152}
 153
 154static int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
 155                         int cq_num, u32 opmod)
 156{
 157        return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ,
 158                        MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
 159}
 160
 161static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
 162                         int cq_num)
 163{
 164        return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0,
 165                            cq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ,
 166                            MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
 167}
 168
 169int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq,
 170                   u16 count, u16 period)
 171{
 172        struct mlx4_cmd_mailbox *mailbox;
 173        struct mlx4_cq_context *cq_context;
 174        int err;
 175
 176        mailbox = mlx4_alloc_cmd_mailbox(dev);
 177        if (IS_ERR(mailbox))
 178                return PTR_ERR(mailbox);
 179
 180        cq_context = mailbox->buf;
 181        cq_context->cq_max_count = cpu_to_be16(count);
 182        cq_context->cq_period    = cpu_to_be16(period);
 183
 184        err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1);
 185
 186        mlx4_free_cmd_mailbox(dev, mailbox);
 187        return err;
 188}
 189EXPORT_SYMBOL_GPL(mlx4_cq_modify);
 190
 191int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,
 192                   int entries, struct mlx4_mtt *mtt)
 193{
 194        struct mlx4_cmd_mailbox *mailbox;
 195        struct mlx4_cq_context *cq_context;
 196        u64 mtt_addr;
 197        int err;
 198
 199        mailbox = mlx4_alloc_cmd_mailbox(dev);
 200        if (IS_ERR(mailbox))
 201                return PTR_ERR(mailbox);
 202
 203        cq_context = mailbox->buf;
 204        cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24);
 205        cq_context->log_page_size   = mtt->page_shift - 12;
 206        mtt_addr = mlx4_mtt_addr(dev, mtt);
 207        cq_context->mtt_base_addr_h = mtt_addr >> 32;
 208        cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
 209
 210        err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0);
 211
 212        mlx4_free_cmd_mailbox(dev, mailbox);
 213        return err;
 214}
 215EXPORT_SYMBOL_GPL(mlx4_cq_resize);
 216
 217int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
 218{
 219        struct mlx4_priv *priv = mlx4_priv(dev);
 220        struct mlx4_cq_table *cq_table = &priv->cq_table;
 221        int err;
 222
 223        *cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
 224        if (*cqn == -1)
 225                return -ENOMEM;
 226
 227        err = mlx4_table_get(dev, &cq_table->table, *cqn);
 228        if (err)
 229                goto err_out;
 230
 231        err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn);
 232        if (err)
 233                goto err_put;
 234        return 0;
 235
 236err_put:
 237        mlx4_table_put(dev, &cq_table->table, *cqn);
 238
 239err_out:
 240        mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR);
 241        return err;
 242}
 243
 244static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn, u8 usage)
 245{
 246        u32 in_modifier = RES_CQ | (((u32)usage & 3) << 30);
 247        u64 out_param;
 248        int err;
 249
 250        if (mlx4_is_mfunc(dev)) {
 251                err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
 252                                   RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,
 253                                   MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
 254                if (err)
 255                        return err;
 256                else {
 257                        *cqn = get_param_l(&out_param);
 258                        return 0;
 259                }
 260        }
 261        return __mlx4_cq_alloc_icm(dev, cqn);
 262}
 263
 264void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
 265{
 266        struct mlx4_priv *priv = mlx4_priv(dev);
 267        struct mlx4_cq_table *cq_table = &priv->cq_table;
 268
 269        mlx4_table_put(dev, &cq_table->cmpt_table, cqn);
 270        mlx4_table_put(dev, &cq_table->table, cqn);
 271        mlx4_bitmap_free(&cq_table->bitmap, cqn, MLX4_NO_RR);
 272}
 273
 274static void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
 275{
 276        u64 in_param = 0;
 277        int err;
 278
 279        if (mlx4_is_mfunc(dev)) {
 280                set_param_l(&in_param, cqn);
 281                err = mlx4_cmd(dev, in_param, RES_CQ, RES_OP_RESERVE_AND_MAP,
 282                               MLX4_CMD_FREE_RES,
 283                               MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
 284                if (err)
 285                        mlx4_warn(dev, "Failed freeing cq:%d\n", cqn);
 286        } else
 287                __mlx4_cq_free_icm(dev, cqn);
 288}
 289
 290static int mlx4_init_user_cqes(void *buf, int entries, int cqe_size)
 291{
 292        int entries_per_copy = PAGE_SIZE / cqe_size;
 293        void *init_ents;
 294        int err = 0;
 295        int i;
 296
 297        init_ents = kmalloc(PAGE_SIZE, GFP_KERNEL);
 298        if (!init_ents)
 299                return -ENOMEM;
 300
 301        /* Populate a list of CQ entries to reduce the number of
 302         * copy_to_user calls. 0xcc is the initialization value
 303         * required by the FW.
 304         */
 305        memset(init_ents, 0xcc, PAGE_SIZE);
 306
 307        if (entries_per_copy < entries) {
 308                for (i = 0; i < entries / entries_per_copy; i++) {
 309                        err = copy_to_user((void __user *)buf, init_ents, PAGE_SIZE) ?
 310                                -EFAULT : 0;
 311                        if (err)
 312                                goto out;
 313
 314                        buf += PAGE_SIZE;
 315                }
 316        } else {
 317                err = copy_to_user((void __user *)buf, init_ents, entries * cqe_size) ?
 318                        -EFAULT : 0;
 319        }
 320
 321out:
 322        kfree(init_ents);
 323
 324        return err;
 325}
 326
 327static void mlx4_init_kernel_cqes(struct mlx4_buf *buf,
 328                                  int entries,
 329                                  int cqe_size)
 330{
 331        int i;
 332
 333        if (buf->nbufs == 1)
 334                memset(buf->direct.buf, 0xcc, entries * cqe_size);
 335        else
 336                for (i = 0; i < buf->npages; i++)
 337                        memset(buf->page_list[i].buf, 0xcc,
 338                               1UL << buf->page_shift);
 339}
 340
 341int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
 342                  struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec,
 343                  struct mlx4_cq *cq, unsigned vector, int collapsed,
 344                  int timestamp_en, void *buf_addr, bool user_cq)
 345{
 346        bool sw_cq_init = dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SW_CQ_INIT;
 347        struct mlx4_priv *priv = mlx4_priv(dev);
 348        struct mlx4_cq_table *cq_table = &priv->cq_table;
 349        struct mlx4_cmd_mailbox *mailbox;
 350        struct mlx4_cq_context *cq_context;
 351        u64 mtt_addr;
 352        int err;
 353
 354        if (vector >= dev->caps.num_comp_vectors)
 355                return -EINVAL;
 356
 357        cq->vector = vector;
 358
 359        err = mlx4_cq_alloc_icm(dev, &cq->cqn, cq->usage);
 360        if (err)
 361                return err;
 362
 363        spin_lock(&cq_table->lock);
 364        err = radix_tree_insert(&cq_table->tree, cq->cqn, cq);
 365        spin_unlock(&cq_table->lock);
 366        if (err)
 367                goto err_icm;
 368
 369        mailbox = mlx4_alloc_cmd_mailbox(dev);
 370        if (IS_ERR(mailbox)) {
 371                err = PTR_ERR(mailbox);
 372                goto err_radix;
 373        }
 374
 375        cq_context = mailbox->buf;
 376        cq_context->flags           = cpu_to_be32(!!collapsed << 18);
 377        if (timestamp_en)
 378                cq_context->flags  |= cpu_to_be32(1 << 19);
 379
 380        cq_context->logsize_usrpage =
 381                cpu_to_be32((ilog2(nent) << 24) |
 382                            mlx4_to_hw_uar_index(dev, uar->index));
 383        cq_context->comp_eqn        = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].eqn;
 384        cq_context->log_page_size   = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
 385
 386        mtt_addr = mlx4_mtt_addr(dev, mtt);
 387        cq_context->mtt_base_addr_h = mtt_addr >> 32;
 388        cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
 389        cq_context->db_rec_addr     = cpu_to_be64(db_rec);
 390
 391        if (sw_cq_init) {
 392                if (user_cq) {
 393                        err = mlx4_init_user_cqes(buf_addr, nent,
 394                                                  dev->caps.cqe_size);
 395                        if (err)
 396                                sw_cq_init = false;
 397                } else {
 398                        mlx4_init_kernel_cqes(buf_addr, nent,
 399                                              dev->caps.cqe_size);
 400                }
 401        }
 402
 403        err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn, sw_cq_init);
 404
 405        mlx4_free_cmd_mailbox(dev, mailbox);
 406        if (err)
 407                goto err_radix;
 408
 409        cq->cons_index = 0;
 410        cq->arm_sn     = 1;
 411        cq->uar        = uar;
 412        refcount_set(&cq->refcount, 1);
 413        init_completion(&cq->free);
 414        cq->comp = mlx4_add_cq_to_tasklet;
 415        cq->tasklet_ctx.priv =
 416                &priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].tasklet_ctx;
 417        INIT_LIST_HEAD(&cq->tasklet_ctx.list);
 418
 419
 420        cq->irq = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].irq;
 421        return 0;
 422
 423err_radix:
 424        spin_lock(&cq_table->lock);
 425        radix_tree_delete(&cq_table->tree, cq->cqn);
 426        spin_unlock(&cq_table->lock);
 427
 428err_icm:
 429        mlx4_cq_free_icm(dev, cq->cqn);
 430
 431        return err;
 432}
 433EXPORT_SYMBOL_GPL(mlx4_cq_alloc);
 434
 435void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
 436{
 437        struct mlx4_priv *priv = mlx4_priv(dev);
 438        struct mlx4_cq_table *cq_table = &priv->cq_table;
 439        int err;
 440
 441        err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn);
 442        if (err)
 443                mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn);
 444
 445        spin_lock(&cq_table->lock);
 446        radix_tree_delete(&cq_table->tree, cq->cqn);
 447        spin_unlock(&cq_table->lock);
 448
 449        synchronize_irq(priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq);
 450        if (priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq !=
 451            priv->eq_table.eq[MLX4_EQ_ASYNC].irq)
 452                synchronize_irq(priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
 453
 454        if (refcount_dec_and_test(&cq->refcount))
 455                complete(&cq->free);
 456        wait_for_completion(&cq->free);
 457
 458        mlx4_cq_free_icm(dev, cq->cqn);
 459}
 460EXPORT_SYMBOL_GPL(mlx4_cq_free);
 461
 462int mlx4_init_cq_table(struct mlx4_dev *dev)
 463{
 464        struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
 465        int err;
 466
 467        spin_lock_init(&cq_table->lock);
 468        INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
 469        if (mlx4_is_slave(dev))
 470                return 0;
 471
 472        err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs,
 473                               dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0);
 474        if (err)
 475                return err;
 476
 477        return 0;
 478}
 479
 480void mlx4_cleanup_cq_table(struct mlx4_dev *dev)
 481{
 482        if (mlx4_is_slave(dev))
 483                return;
 484        /* Nothing to do to clean up radix_tree */
 485        mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap);
 486}
 487