1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37#include <linux/hardirq.h>
38
39#include <linux/mlx4/cmd.h>
40#include <linux/mlx4/cq.h>
41
42#include "mlx4.h"
43#include "icm.h"
44
45struct mlx4_cq_context {
46 __be32 flags;
47 u16 reserved1[3];
48 __be16 page_offset;
49 __be32 logsize_usrpage;
50 __be16 cq_period;
51 __be16 cq_max_count;
52 u8 reserved2[3];
53 u8 comp_eqn;
54 u8 log_page_size;
55 u8 reserved3[2];
56 u8 mtt_base_addr_h;
57 __be32 mtt_base_addr_l;
58 __be32 last_notified_index;
59 __be32 solicit_producer_index;
60 __be32 consumer_index;
61 __be32 producer_index;
62 u32 reserved4[2];
63 __be64 db_rec_addr;
64};
65
66#define MLX4_CQ_STATUS_OK ( 0 << 28)
67#define MLX4_CQ_STATUS_OVERFLOW ( 9 << 28)
68#define MLX4_CQ_STATUS_WRITE_FAIL (10 << 28)
69#define MLX4_CQ_FLAG_CC ( 1 << 18)
70#define MLX4_CQ_FLAG_OI ( 1 << 17)
71#define MLX4_CQ_STATE_ARMED ( 9 << 8)
72#define MLX4_CQ_STATE_ARMED_SOL ( 6 << 8)
73#define MLX4_EQ_STATE_FIRED (10 << 8)
74
75void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn)
76{
77 struct mlx4_cq *cq;
78
79 cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree,
80 cqn & (dev->caps.num_cqs - 1));
81 if (!cq) {
82 mlx4_warn(dev, "Completion event for bogus CQ %08x\n", cqn);
83 return;
84 }
85
86 ++cq->arm_sn;
87
88 cq->comp(cq);
89}
90
91void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type)
92{
93 struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
94 struct mlx4_cq *cq;
95
96 spin_lock(&cq_table->lock);
97
98 cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1));
99 if (cq)
100 atomic_inc(&cq->refcount);
101
102 spin_unlock(&cq_table->lock);
103
104 if (!cq) {
105 mlx4_warn(dev, "Async event for bogus CQ %08x\n", cqn);
106 return;
107 }
108
109 cq->event(cq, event_type);
110
111 if (atomic_dec_and_test(&cq->refcount))
112 complete(&cq->free);
113}
114
115static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
116 int cq_num)
117{
118 return mlx4_cmd(dev, mailbox->dma, cq_num, 0, MLX4_CMD_SW2HW_CQ,
119 MLX4_CMD_TIME_CLASS_A);
120}
121
122static int mlx4_MODIFY_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
123 int cq_num, u32 opmod)
124{
125 return mlx4_cmd(dev, mailbox->dma, cq_num, opmod, MLX4_CMD_MODIFY_CQ,
126 MLX4_CMD_TIME_CLASS_A);
127}
128
129static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
130 int cq_num)
131{
132 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, cq_num,
133 mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ,
134 MLX4_CMD_TIME_CLASS_A);
135}
136
137int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq,
138 u16 count, u16 period)
139{
140 struct mlx4_cmd_mailbox *mailbox;
141 struct mlx4_cq_context *cq_context;
142 int err;
143
144 mailbox = mlx4_alloc_cmd_mailbox(dev);
145 if (IS_ERR(mailbox))
146 return PTR_ERR(mailbox);
147
148 cq_context = mailbox->buf;
149 memset(cq_context, 0, sizeof *cq_context);
150
151 cq_context->cq_max_count = cpu_to_be16(count);
152 cq_context->cq_period = cpu_to_be16(period);
153
154 err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1);
155
156 mlx4_free_cmd_mailbox(dev, mailbox);
157 return err;
158}
159EXPORT_SYMBOL_GPL(mlx4_cq_modify);
160
161int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,
162 int entries, struct mlx4_mtt *mtt)
163{
164 struct mlx4_cmd_mailbox *mailbox;
165 struct mlx4_cq_context *cq_context;
166 u64 mtt_addr;
167 int err;
168
169 mailbox = mlx4_alloc_cmd_mailbox(dev);
170 if (IS_ERR(mailbox))
171 return PTR_ERR(mailbox);
172
173 cq_context = mailbox->buf;
174 memset(cq_context, 0, sizeof *cq_context);
175
176 cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24);
177 cq_context->log_page_size = mtt->page_shift - 12;
178 mtt_addr = mlx4_mtt_addr(dev, mtt);
179 cq_context->mtt_base_addr_h = mtt_addr >> 32;
180 cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
181
182 err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 0);
183
184 mlx4_free_cmd_mailbox(dev, mailbox);
185 return err;
186}
187EXPORT_SYMBOL_GPL(mlx4_cq_resize);
188
189int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,
190 struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq,
191 unsigned vector, int collapsed)
192{
193 struct mlx4_priv *priv = mlx4_priv(dev);
194 struct mlx4_cq_table *cq_table = &priv->cq_table;
195 struct mlx4_cmd_mailbox *mailbox;
196 struct mlx4_cq_context *cq_context;
197 u64 mtt_addr;
198 int err;
199
200 if (vector >= dev->caps.num_comp_vectors)
201 return -EINVAL;
202
203 cq->vector = vector;
204
205 cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap);
206 if (cq->cqn == -1)
207 return -ENOMEM;
208
209 err = mlx4_table_get(dev, &cq_table->table, cq->cqn);
210 if (err)
211 goto err_out;
212
213 err = mlx4_table_get(dev, &cq_table->cmpt_table, cq->cqn);
214 if (err)
215 goto err_put;
216
217 spin_lock_irq(&cq_table->lock);
218 err = radix_tree_insert(&cq_table->tree, cq->cqn, cq);
219 spin_unlock_irq(&cq_table->lock);
220 if (err)
221 goto err_cmpt_put;
222
223 mailbox = mlx4_alloc_cmd_mailbox(dev);
224 if (IS_ERR(mailbox)) {
225 err = PTR_ERR(mailbox);
226 goto err_radix;
227 }
228
229 cq_context = mailbox->buf;
230 memset(cq_context, 0, sizeof *cq_context);
231
232 cq_context->flags = cpu_to_be32(!!collapsed << 18);
233 cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index);
234 cq_context->comp_eqn = priv->eq_table.eq[vector].eqn;
235 cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
236
237 mtt_addr = mlx4_mtt_addr(dev, mtt);
238 cq_context->mtt_base_addr_h = mtt_addr >> 32;
239 cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
240 cq_context->db_rec_addr = cpu_to_be64(db_rec);
241
242 err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn);
243 mlx4_free_cmd_mailbox(dev, mailbox);
244 if (err)
245 goto err_radix;
246
247 cq->cons_index = 0;
248 cq->arm_sn = 1;
249 cq->uar = uar;
250 atomic_set(&cq->refcount, 1);
251 init_completion(&cq->free);
252
253 return 0;
254
255err_radix:
256 spin_lock_irq(&cq_table->lock);
257 radix_tree_delete(&cq_table->tree, cq->cqn);
258 spin_unlock_irq(&cq_table->lock);
259
260err_cmpt_put:
261 mlx4_table_put(dev, &cq_table->cmpt_table, cq->cqn);
262
263err_put:
264 mlx4_table_put(dev, &cq_table->table, cq->cqn);
265
266err_out:
267 mlx4_bitmap_free(&cq_table->bitmap, cq->cqn);
268
269 return err;
270}
271EXPORT_SYMBOL_GPL(mlx4_cq_alloc);
272
273void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
274{
275 struct mlx4_priv *priv = mlx4_priv(dev);
276 struct mlx4_cq_table *cq_table = &priv->cq_table;
277 int err;
278
279 err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn);
280 if (err)
281 mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn);
282
283 synchronize_irq(priv->eq_table.eq[cq->vector].irq);
284
285 spin_lock_irq(&cq_table->lock);
286 radix_tree_delete(&cq_table->tree, cq->cqn);
287 spin_unlock_irq(&cq_table->lock);
288
289 if (atomic_dec_and_test(&cq->refcount))
290 complete(&cq->free);
291 wait_for_completion(&cq->free);
292
293 mlx4_table_put(dev, &cq_table->table, cq->cqn);
294 mlx4_bitmap_free(&cq_table->bitmap, cq->cqn);
295}
296EXPORT_SYMBOL_GPL(mlx4_cq_free);
297
298int mlx4_init_cq_table(struct mlx4_dev *dev)
299{
300 struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
301 int err;
302
303 spin_lock_init(&cq_table->lock);
304 INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
305
306 err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs,
307 dev->caps.num_cqs - 1, dev->caps.reserved_cqs, 0);
308 if (err)
309 return err;
310
311 return 0;
312}
313
314void mlx4_cleanup_cq_table(struct mlx4_dev *dev)
315{
316
317 mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap);
318}
319
