1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37#include <linux/init.h>
38#include <linux/hardirq.h>
39#include <linux/export.h>
40
41#include <linux/mlx4/cmd.h>
42#include <linux/mlx4/cq.h>
43
44#include "mlx4.h"
45#include "icm.h"
46
47#define MLX4_CQ_STATUS_OK ( 0 << 28)
48#define MLX4_CQ_STATUS_OVERFLOW ( 9 << 28)
49#define MLX4_CQ_STATUS_WRITE_FAIL (10 << 28)
50#define MLX4_CQ_FLAG_CC ( 1 << 18)
51#define MLX4_CQ_FLAG_OI ( 1 << 17)
52#define MLX4_CQ_STATE_ARMED ( 9 << 8)
53#define MLX4_CQ_STATE_ARMED_SOL ( 6 << 8)
54#define MLX4_EQ_STATE_FIRED (10 << 8)
55
56#define TASKLET_MAX_TIME 2
57#define TASKLET_MAX_TIME_JIFFIES msecs_to_jiffies(TASKLET_MAX_TIME)
58
59void mlx4_cq_tasklet_cb(unsigned long data)
60{
61 unsigned long flags;
62 unsigned long end = jiffies + TASKLET_MAX_TIME_JIFFIES;
63 struct mlx4_eq_tasklet *ctx = (struct mlx4_eq_tasklet *)data;
64 struct mlx4_cq *mcq, *temp;
65
66 spin_lock_irqsave(&ctx->lock, flags);
67 list_splice_tail_init(&ctx->list, &ctx->process_list);
68 spin_unlock_irqrestore(&ctx->lock, flags);
69
70 list_for_each_entry_safe(mcq, temp, &ctx->process_list, tasklet_ctx.list) {
71 list_del_init(&mcq->tasklet_ctx.list);
72 mcq->tasklet_ctx.comp(mcq);
73 if (atomic_dec_and_test(&mcq->refcount))
74 complete(&mcq->free);
75 if (time_after(jiffies, end))
76 break;
77 }
78
79 if (!list_empty(&ctx->process_list))
80 tasklet_schedule(&ctx->task);
81}
82
83static void mlx4_add_cq_to_tasklet(struct mlx4_cq *cq)
84{
85 struct mlx4_eq_tasklet *tasklet_ctx = cq->tasklet_ctx.priv;
86 unsigned long flags;
87 bool kick;
88
89 spin_lock_irqsave(&tasklet_ctx->lock, flags);
90
91
92
93
94
95 if (list_empty_careful(&cq->tasklet_ctx.list)) {
96 atomic_inc(&cq->refcount);
97 kick = list_empty(&tasklet_ctx->list);
98 list_add_tail(&cq->tasklet_ctx.list, &tasklet_ctx->list);
99 if (kick)
100 tasklet_schedule(&tasklet_ctx->task);
101 }
102 spin_unlock_irqrestore(&tasklet_ctx->lock, flags);
103}
104
105void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn)
106{
107 struct mlx4_cq *cq;
108
109 rcu_read_lock();
110 cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree,
111 cqn & (dev->caps.num_cqs - 1));
112 rcu_read_unlock();
113
114 if (!cq) {
115 mlx4_dbg(dev, "Completion event for bogus CQ %08x\n", cqn);
116 return;
117 }
118
119
120
121
122 ++cq->arm_sn;
123
124 cq->comp(cq);
125}
126
127void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type)
128{
129 struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
130 struct mlx4_cq *cq;
131
132 rcu_read_lock();
133 cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1));
134 rcu_read_unlock();
135
136 if (!cq) {
137 mlx4_dbg(dev, "Async event for bogus CQ %08x\n", cqn);
138 return;
139 }
140
141
142
143
144 cq->event(cq, event_type);
145}
146
147static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
148 int cq_num)
149{
150 return mlx4_cmd(dev, mailbox->dma, cq_num, 0,
151 MLX4_CMD_SW2HW_CQ, MLX4_CMD_TIME_CLASS_A,
152 MLX4_CMD_WRAPPED);
153}
154
155static int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
156 int cq_num, u32 opmod)
157{
158 return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ,
159 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
160}
161
162static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
163 int cq_num)
164{
165 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0,
166 cq_num, mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ,
167 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
168}
169
170int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq,
171 u16 count, u16 period)
172{
173 struct mlx4_cmd_mailbox *mailbox;
174 struct mlx4_cq_context *cq_context;
175 int err;
176
177 mailbox = mlx4_alloc_cmd_mailbox(dev);
178 if (IS_ERR(mailbox))
179 return PTR_ERR(mailbox);
180
181 cq_context = mailbox->buf;
182 cq_context->cq_max_count = cpu_to_be16(count);
183 cq_context->cq_period = cpu_to_be16(period);
184
185 err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1);
186
187 mlx4_free_cmd_mailbox(dev, mailbox);
188 return err;
189}
190EXPORT_SYMBOL_GPL(mlx4_cq_modify);
191
192int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,
193 int entries, struct mlx4_mtt *mtt)
194{
195 struct mlx4_cmd_mailbox *mailbox;
196 struct mlx4_cq_context *cq_context;
197 u64 mtt_addr;
198 int err;
199
200 mailbox = mlx4_alloc_cmd_mailbox(dev);
201 if (IS_ERR(mailbox))
202 return PTR_ERR(mailbox);
203
204 cq_context = mailbox->buf;
205 cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24);
206 cq_context->log_page_size = mtt->page_shift - 12;
207 mtt_addr = mlx4_mtt_addr(dev, mtt);
208 cq_context->mtt_base_addr_h = mtt_addr >> 32;
209 cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
210
211 err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0);
212
213 mlx4_free_cmd_mailbox(dev, mailbox);
214 return err;
215}
216EXPORT_SYMBOL_GPL(mlx4_cq_resize);
217
218int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
219{
220 struct mlx4_priv *priv = mlx4_priv(dev);
221 struct mlx4_cq_table *cq_table = &priv->cq_table;
222 int err;
223
224 *cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
225 if (*cqn == -1)
226 return -ENOMEM;
227
228 err = mlx4_table_get(dev, &cq_table->table, *cqn);
229 if (err)
230 goto err_out;
231
232 err = mlx4_table_get(dev, &cq_table->cmpt_table, *cqn);
233 if (err)
234 goto err_put;
235 return 0;
236
237err_put:
238 mlx4_table_put(dev, &cq_table->table, *cqn);
239
240err_out:
241 mlx4_bitmap_free(&cq_table->bitmap, *cqn, MLX4_NO_RR);
242 return err;
243}
244
245static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn, u8 usage)
246{
247 u32 in_modifier = RES_CQ | (((u32)usage & 3) << 30);
248 u64 out_param;
249 int err;
250
251 if (mlx4_is_mfunc(dev)) {
252 err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
253 RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,
254 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
255 if (err)
256 return err;
257 else {
258 *cqn = get_param_l(&out_param);
259 return 0;
260 }
261 }
262 return __mlx4_cq_alloc_icm(dev, cqn);
263}
264
265void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
266{
267 struct mlx4_priv *priv = mlx4_priv(dev);
268 struct mlx4_cq_table *cq_table = &priv->cq_table;
269
270 mlx4_table_put(dev, &cq_table->cmpt_table, cqn);
271 mlx4_table_put(dev, &cq_table->table, cqn);
272 mlx4_bitmap_free(&cq_table->bitmap, cqn, MLX4_NO_RR);
273}
274
275static void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
276{
277 u64 in_param = 0;
278 int err;
279
280 if (mlx4_is_mfunc(dev)) {
281 set_param_l(&in_param, cqn);
282 err = mlx4_cmd(dev, in_param, RES_CQ, RES_OP_RESERVE_AND_MAP,
283 MLX4_CMD_FREE_RES,
284 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
285 if (err)
286 mlx4_warn(dev, "Failed freeing cq:%d\n", cqn);
287 } else
288 __mlx4_cq_free_icm(dev, cqn);
289}
290
291int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
292 struct mlx4_mtt *mtt, struct mlx4_uar *uar, u64 db_rec,
293 struct mlx4_cq *cq, unsigned vector, int collapsed,
294 int timestamp_en)
295{
296 struct mlx4_priv *priv = mlx4_priv(dev);
297 struct mlx4_cq_table *cq_table = &priv->cq_table;
298 struct mlx4_cmd_mailbox *mailbox;
299 struct mlx4_cq_context *cq_context;
300 u64 mtt_addr;
301 int err;
302
303 if (vector >= dev->caps.num_comp_vectors)
304 return -EINVAL;
305
306 cq->vector = vector;
307
308 err = mlx4_cq_alloc_icm(dev, &cq->cqn, cq->usage);
309 if (err)
310 return err;
311
312 spin_lock(&cq_table->lock);
313 err = radix_tree_insert(&cq_table->tree, cq->cqn, cq);
314 spin_unlock(&cq_table->lock);
315 if (err)
316 goto err_icm;
317
318 mailbox = mlx4_alloc_cmd_mailbox(dev);
319 if (IS_ERR(mailbox)) {
320 err = PTR_ERR(mailbox);
321 goto err_radix;
322 }
323
324 cq_context = mailbox->buf;
325 cq_context->flags = cpu_to_be32(!!collapsed << 18);
326 if (timestamp_en)
327 cq_context->flags |= cpu_to_be32(1 << 19);
328
329 cq_context->logsize_usrpage =
330 cpu_to_be32((ilog2(nent) << 24) |
331 mlx4_to_hw_uar_index(dev, uar->index));
332 cq_context->comp_eqn = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].eqn;
333 cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
334
335 mtt_addr = mlx4_mtt_addr(dev, mtt);
336 cq_context->mtt_base_addr_h = mtt_addr >> 32;
337 cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
338 cq_context->db_rec_addr = cpu_to_be64(db_rec);
339
340 err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn);
341 mlx4_free_cmd_mailbox(dev, mailbox);
342 if (err)
343 goto err_radix;
344
345 cq->cons_index = 0;
346 cq->arm_sn = 1;
347 cq->uar = uar;
348 atomic_set(&cq->refcount, 1);
349 init_completion(&cq->free);
350 cq->comp = mlx4_add_cq_to_tasklet;
351 cq->tasklet_ctx.priv =
352 &priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].tasklet_ctx;
353 INIT_LIST_HEAD(&cq->tasklet_ctx.list);
354
355
356 cq->irq = priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].irq;
357 return 0;
358
359err_radix:
360 spin_lock(&cq_table->lock);
361 radix_tree_delete(&cq_table->tree, cq->cqn);
362 spin_unlock(&cq_table->lock);
363
364err_icm:
365 mlx4_cq_free_icm(dev, cq->cqn);
366
367 return err;
368}
369EXPORT_SYMBOL_GPL(mlx4_cq_alloc);
370
371void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
372{
373 struct mlx4_priv *priv = mlx4_priv(dev);
374 struct mlx4_cq_table *cq_table = &priv->cq_table;
375 int err;
376
377 err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn);
378 if (err)
379 mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn);
380
381 spin_lock(&cq_table->lock);
382 radix_tree_delete(&cq_table->tree, cq->cqn);
383 spin_unlock(&cq_table->lock);
384
385 synchronize_irq(priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq);
386 if (priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq->vector)].irq !=
387 priv->eq_table.eq[MLX4_EQ_ASYNC].irq)
388 synchronize_irq(priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
389
390 if (atomic_dec_and_test(&cq->refcount))
391 complete(&cq->free);
392 wait_for_completion(&cq->free);
393
394 mlx4_cq_free_icm(dev, cq->cqn);
395}
396EXPORT_SYMBOL_GPL(mlx4_cq_free);
397
398int mlx4_init_cq_table(struct mlx4_dev *dev)
399{
400 struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
401 int err;
402
403 spin_lock_init(&cq_table->lock);
404 INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
405 if (mlx4_is_slave(dev))
406 return 0;
407
408 err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs,
409 dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0);
410 if (err)
411 return err;
412
413 return 0;
414}
415
416void mlx4_cleanup_cq_table(struct mlx4_dev *dev)
417{
418 if (mlx4_is_slave(dev))
419 return;
420
421 mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap);
422}
423
