1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/log2.h>
35#include <linux/slab.h>
36#include <linux/netdevice.h>
37
38#include <rdma/ib_cache.h>
39#include <rdma/ib_pack.h>
40#include <rdma/ib_addr.h>
41
42#include <linux/mlx4/qp.h>
43
44#include "mlx4_ib.h"
45#include "user.h"
46
47enum {
48 MLX4_IB_ACK_REQ_FREQ = 8,
49};
50
51enum {
52 MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,
53 MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
54 MLX4_IB_LINK_TYPE_IB = 0,
55 MLX4_IB_LINK_TYPE_ETH = 1
56};
57
58enum {
59
60
61
62
63
64
65 MLX4_IB_UD_HEADER_SIZE = 82,
66 MLX4_IB_LSO_HEADER_SPARE = 128,
67};
68
69enum {
70 MLX4_IB_IBOE_ETHERTYPE = 0x8915
71};
72
73struct mlx4_ib_sqp {
74 struct mlx4_ib_qp qp;
75 int pkey_index;
76 u32 qkey;
77 u32 send_psn;
78 struct ib_ud_header ud_header;
79 u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
80};
81
82enum {
83 MLX4_IB_MIN_SQ_STRIDE = 6,
84 MLX4_IB_CACHE_LINE_SIZE = 64,
85};
86
87static const __be32 mlx4_ib_opcode[] = {
88 [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND),
89 [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO),
90 [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM),
91 [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
92 [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
93 [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ),
94 [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
95 [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
96 [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
97 [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
98 [IB_WR_FAST_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR),
99 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
100 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
101};
102
103static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
104{
105 return container_of(mqp, struct mlx4_ib_sqp, qp);
106}
107
108static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
109{
110 return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
111 qp->mqp.qpn <= dev->dev->caps.sqp_start + 3;
112}
113
114static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
115{
116 return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
117 qp->mqp.qpn <= dev->dev->caps.sqp_start + 1;
118}
119
120static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
121{
122 return mlx4_buf_offset(&qp->buf, offset);
123}
124
125static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
126{
127 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
128}
129
130static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
131{
132 return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
133}
134
135
136
137
138
139
140
141
142
143
144static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
145{
146 __be32 *wqe;
147 int i;
148 int s;
149 int ind;
150 void *buf;
151 __be32 stamp;
152 struct mlx4_wqe_ctrl_seg *ctrl;
153
154 if (qp->sq_max_wqes_per_wr > 1) {
155 s = roundup(size, 1U << qp->sq.wqe_shift);
156 for (i = 0; i < s; i += 64) {
157 ind = (i >> qp->sq.wqe_shift) + n;
158 stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
159 cpu_to_be32(0xffffffff);
160 buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
161 wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
162 *wqe = stamp;
163 }
164 } else {
165 ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
166 s = (ctrl->fence_size & 0x3f) << 4;
167 for (i = 64; i < s; i += 64) {
168 wqe = buf + i;
169 *wqe = cpu_to_be32(0xffffffff);
170 }
171 }
172}
173
174static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
175{
176 struct mlx4_wqe_ctrl_seg *ctrl;
177 struct mlx4_wqe_inline_seg *inl;
178 void *wqe;
179 int s;
180
181 ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
182 s = sizeof(struct mlx4_wqe_ctrl_seg);
183
184 if (qp->ibqp.qp_type == IB_QPT_UD) {
185 struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
186 struct mlx4_av *av = (struct mlx4_av *)dgram->av;
187 memset(dgram, 0, sizeof *dgram);
188 av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
189 s += sizeof(struct mlx4_wqe_datagram_seg);
190 }
191
192
193 if (size > s) {
194 inl = wqe + s;
195 inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
196 }
197 ctrl->srcrb_flags = 0;
198 ctrl->fence_size = size / 16;
199
200
201
202
203 wmb();
204
205 ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
206 (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
207
208 stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
209}
210
211
212static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
213{
214 unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
215 if (unlikely(s < qp->sq_max_wqes_per_wr)) {
216 post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
217 ind += s;
218 }
219 return ind;
220}
221
222static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
223{
224 struct ib_event event;
225 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
226
227 if (type == MLX4_EVENT_TYPE_PATH_MIG)
228 to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
229
230 if (ibqp->event_handler) {
231 event.device = ibqp->device;
232 event.element.qp = ibqp;
233 switch (type) {
234 case MLX4_EVENT_TYPE_PATH_MIG:
235 event.event = IB_EVENT_PATH_MIG;
236 break;
237 case MLX4_EVENT_TYPE_COMM_EST:
238 event.event = IB_EVENT_COMM_EST;
239 break;
240 case MLX4_EVENT_TYPE_SQ_DRAINED:
241 event.event = IB_EVENT_SQ_DRAINED;
242 break;
243 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
244 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
245 break;
246 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
247 event.event = IB_EVENT_QP_FATAL;
248 break;
249 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
250 event.event = IB_EVENT_PATH_MIG_ERR;
251 break;
252 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
253 event.event = IB_EVENT_QP_REQ_ERR;
254 break;
255 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
256 event.event = IB_EVENT_QP_ACCESS_ERR;
257 break;
258 default:
259 printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
260 "on QP %06x\n", type, qp->qpn);
261 return;
262 }
263
264 ibqp->event_handler(&event, ibqp->qp_context);
265 }
266}
267
268static int send_wqe_overhead(enum ib_qp_type type, u32 flags)
269{
270
271
272
273
274
275
276 switch (type) {
277 case IB_QPT_UD:
278 return sizeof (struct mlx4_wqe_ctrl_seg) +
279 sizeof (struct mlx4_wqe_datagram_seg) +
280 ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
281 case IB_QPT_UC:
282 return sizeof (struct mlx4_wqe_ctrl_seg) +
283 sizeof (struct mlx4_wqe_raddr_seg);
284 case IB_QPT_RC:
285 return sizeof (struct mlx4_wqe_ctrl_seg) +
286 sizeof (struct mlx4_wqe_atomic_seg) +
287 sizeof (struct mlx4_wqe_raddr_seg);
288 case IB_QPT_SMI:
289 case IB_QPT_GSI:
290 return sizeof (struct mlx4_wqe_ctrl_seg) +
291 ALIGN(MLX4_IB_UD_HEADER_SIZE +
292 DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
293 MLX4_INLINE_ALIGN) *
294 sizeof (struct mlx4_wqe_inline_seg),
295 sizeof (struct mlx4_wqe_data_seg)) +
296 ALIGN(4 +
297 sizeof (struct mlx4_wqe_inline_seg),
298 sizeof (struct mlx4_wqe_data_seg));
299 default:
300 return sizeof (struct mlx4_wqe_ctrl_seg);
301 }
302}
303
304static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
305 int is_user, int has_srq, struct mlx4_ib_qp *qp)
306{
307
308 if (cap->max_recv_wr > dev->dev->caps.max_wqes ||
309 cap->max_recv_sge > dev->dev->caps.max_rq_sg)
310 return -EINVAL;
311
312 if (has_srq) {
313
314 if (cap->max_recv_wr)
315 return -EINVAL;
316
317 qp->rq.wqe_cnt = qp->rq.max_gs = 0;
318 } else {
319
320 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
321 return -EINVAL;
322
323 qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));
324 qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));
325 qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
326 }
327
328 cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
329 cap->max_recv_sge = qp->rq.max_gs;
330
331 return 0;
332}
333
334static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
335 enum ib_qp_type type, struct mlx4_ib_qp *qp)
336{
337 int s;
338
339
340 if (cap->max_send_wr > dev->dev->caps.max_wqes ||
341 cap->max_send_sge > dev->dev->caps.max_sq_sg ||
342 cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
343 sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
344 return -EINVAL;
345
346
347
348
349
350 if ((type == IB_QPT_SMI || type == IB_QPT_GSI) &&
351 cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
352 return -EINVAL;
353
354 s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
355 cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
356 send_wqe_overhead(type, qp->flags);
357
358 if (s > dev->dev->caps.max_sq_desc_sz)
359 return -EINVAL;
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392 if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
393 qp->sq_signal_bits && BITS_PER_LONG == 64 &&
394 type != IB_QPT_SMI && type != IB_QPT_GSI)
395 qp->sq.wqe_shift = ilog2(64);
396 else
397 qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
398
399 for (;;) {
400 qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
401
402
403
404
405
406 qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
407 qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
408 qp->sq_max_wqes_per_wr +
409 qp->sq_spare_wqes);
410
411 if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
412 break;
413
414 if (qp->sq_max_wqes_per_wr <= 1)
415 return -EINVAL;
416
417 ++qp->sq.wqe_shift;
418 }
419
420 qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
421 (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
422 send_wqe_overhead(type, qp->flags)) /
423 sizeof (struct mlx4_wqe_data_seg);
424
425 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
426 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
427 if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
428 qp->rq.offset = 0;
429 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
430 } else {
431 qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
432 qp->sq.offset = 0;
433 }
434
435 cap->max_send_wr = qp->sq.max_post =
436 (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
437 cap->max_send_sge = min(qp->sq.max_gs,
438 min(dev->dev->caps.max_sq_sg,
439 dev->dev->caps.max_rq_sg));
440
441 cap->max_inline_data = 0;
442
443 return 0;
444}
445
446static int set_user_sq_size(struct mlx4_ib_dev *dev,
447 struct mlx4_ib_qp *qp,
448 struct mlx4_ib_create_qp *ucmd)
449{
450
451 if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||
452 ucmd->log_sq_stride >
453 ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
454 ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
455 return -EINVAL;
456
457 qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
458 qp->sq.wqe_shift = ucmd->log_sq_stride;
459
460 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
461 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
462
463 return 0;
464}
465
466static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
467 struct ib_qp_init_attr *init_attr,
468 struct ib_udata *udata, int sqpn, struct mlx4_ib_qp *qp)
469{
470 int qpn;
471 int err;
472
473 mutex_init(&qp->mutex);
474 spin_lock_init(&qp->sq.lock);
475 spin_lock_init(&qp->rq.lock);
476 INIT_LIST_HEAD(&qp->gid_list);
477
478 qp->state = IB_QPS_RESET;
479 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
480 qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
481
482 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, !!init_attr->srq, qp);
483 if (err)
484 goto err;
485
486 if (pd->uobject) {
487 struct mlx4_ib_create_qp ucmd;
488
489 if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
490 err = -EFAULT;
491 goto err;
492 }
493
494 qp->sq_no_prefetch = ucmd.sq_no_prefetch;
495
496 err = set_user_sq_size(dev, qp, &ucmd);
497 if (err)
498 goto err;
499
500 qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
501 qp->buf_size, 0, 0);
502 if (IS_ERR(qp->umem)) {
503 err = PTR_ERR(qp->umem);
504 goto err;
505 }
506
507 err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem),
508 ilog2(qp->umem->page_size), &qp->mtt);
509 if (err)
510 goto err_buf;
511
512 err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
513 if (err)
514 goto err_mtt;
515
516 if (!init_attr->srq) {
517 err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
518 ucmd.db_addr, &qp->db);
519 if (err)
520 goto err_mtt;
521 }
522 } else {
523 qp->sq_no_prefetch = 0;
524
525 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
526 qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
527
528 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
529 qp->flags |= MLX4_IB_QP_LSO;
530
531 err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp);
532 if (err)
533 goto err;
534
535 if (!init_attr->srq) {
536 err = mlx4_db_alloc(dev->dev, &qp->db, 0);
537 if (err)
538 goto err;
539
540 *qp->db.db = 0;
541 }
542
543 if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) {
544 err = -ENOMEM;
545 goto err_db;
546 }
547
548 err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
549 &qp->mtt);
550 if (err)
551 goto err_buf;
552
553 err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
554 if (err)
555 goto err_mtt;
556
557 qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL);
558 qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL);
559
560 if (!qp->sq.wrid || !qp->rq.wrid) {
561 err = -ENOMEM;
562 goto err_wrid;
563 }
564 }
565
566 if (sqpn) {
567 qpn = sqpn;
568 } else {
569 err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn);
570 if (err)
571 goto err_wrid;
572 }
573
574 err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
575 if (err)
576 goto err_qpn;
577
578
579
580
581
582
583 qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
584
585 qp->mqp.event = mlx4_ib_qp_event;
586
587 return 0;
588
589err_qpn:
590 if (!sqpn)
591 mlx4_qp_release_range(dev->dev, qpn, 1);
592
593err_wrid:
594 if (pd->uobject) {
595 if (!init_attr->srq)
596 mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context),
597 &qp->db);
598 } else {
599 kfree(qp->sq.wrid);
600 kfree(qp->rq.wrid);
601 }
602
603err_mtt:
604 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
605
606err_buf:
607 if (pd->uobject)
608 ib_umem_release(qp->umem);
609 else
610 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
611
612err_db:
613 if (!pd->uobject && !init_attr->srq)
614 mlx4_db_free(dev->dev, &qp->db);
615
616err:
617 return err;
618}
619
620static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
621{
622 switch (state) {
623 case IB_QPS_RESET: return MLX4_QP_STATE_RST;
624 case IB_QPS_INIT: return MLX4_QP_STATE_INIT;
625 case IB_QPS_RTR: return MLX4_QP_STATE_RTR;
626 case IB_QPS_RTS: return MLX4_QP_STATE_RTS;
627 case IB_QPS_SQD: return MLX4_QP_STATE_SQD;
628 case IB_QPS_SQE: return MLX4_QP_STATE_SQER;
629 case IB_QPS_ERR: return MLX4_QP_STATE_ERR;
630 default: return -1;
631 }
632}
633
634static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
635 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
636{
637 if (send_cq == recv_cq) {
638 spin_lock_irq(&send_cq->lock);
639 __acquire(&recv_cq->lock);
640 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
641 spin_lock_irq(&send_cq->lock);
642 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
643 } else {
644 spin_lock_irq(&recv_cq->lock);
645 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
646 }
647}
648
649static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
650 __releases(&send_cq->lock) __releases(&recv_cq->lock)
651{
652 if (send_cq == recv_cq) {
653 __release(&recv_cq->lock);
654 spin_unlock_irq(&send_cq->lock);
655 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
656 spin_unlock(&recv_cq->lock);
657 spin_unlock_irq(&send_cq->lock);
658 } else {
659 spin_unlock(&send_cq->lock);
660 spin_unlock_irq(&recv_cq->lock);
661 }
662}
663
664static void del_gid_entries(struct mlx4_ib_qp *qp)
665{
666 struct mlx4_ib_gid_entry *ge, *tmp;
667
668 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
669 list_del(&ge->list);
670 kfree(ge);
671 }
672}
673
674static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
675 int is_user)
676{
677 struct mlx4_ib_cq *send_cq, *recv_cq;
678
679 if (qp->state != IB_QPS_RESET)
680 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
681 MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
682 printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n",
683 qp->mqp.qpn);
684
685 send_cq = to_mcq(qp->ibqp.send_cq);
686 recv_cq = to_mcq(qp->ibqp.recv_cq);
687
688 mlx4_ib_lock_cqs(send_cq, recv_cq);
689
690 if (!is_user) {
691 __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
692 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
693 if (send_cq != recv_cq)
694 __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
695 }
696
697 mlx4_qp_remove(dev->dev, &qp->mqp);
698
699 mlx4_ib_unlock_cqs(send_cq, recv_cq);
700
701 mlx4_qp_free(dev->dev, &qp->mqp);
702
703 if (!is_sqp(dev, qp))
704 mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
705
706 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
707
708 if (is_user) {
709 if (!qp->ibqp.srq)
710 mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
711 &qp->db);
712 ib_umem_release(qp->umem);
713 } else {
714 kfree(qp->sq.wrid);
715 kfree(qp->rq.wrid);
716 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
717 if (!qp->ibqp.srq)
718 mlx4_db_free(dev->dev, &qp->db);
719 }
720
721 del_gid_entries(qp);
722}
723
724struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
725 struct ib_qp_init_attr *init_attr,
726 struct ib_udata *udata)
727{
728 struct mlx4_ib_dev *dev = to_mdev(pd->device);
729 struct mlx4_ib_sqp *sqp;
730 struct mlx4_ib_qp *qp;
731 int err;
732
733
734
735
736
737 if (init_attr->create_flags & ~(IB_QP_CREATE_IPOIB_UD_LSO |
738 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK))
739 return ERR_PTR(-EINVAL);
740
741 if (init_attr->create_flags &&
742 (pd->uobject || init_attr->qp_type != IB_QPT_UD))
743 return ERR_PTR(-EINVAL);
744
745 switch (init_attr->qp_type) {
746 case IB_QPT_RC:
747 case IB_QPT_UC:
748 case IB_QPT_UD:
749 {
750 qp = kzalloc(sizeof *qp, GFP_KERNEL);
751 if (!qp)
752 return ERR_PTR(-ENOMEM);
753
754 err = create_qp_common(dev, pd, init_attr, udata, 0, qp);
755 if (err) {
756 kfree(qp);
757 return ERR_PTR(err);
758 }
759
760 qp->ibqp.qp_num = qp->mqp.qpn;
761
762 break;
763 }
764 case IB_QPT_SMI:
765 case IB_QPT_GSI:
766 {
767
768 if (pd->uobject)
769 return ERR_PTR(-EINVAL);
770
771 sqp = kzalloc(sizeof *sqp, GFP_KERNEL);
772 if (!sqp)
773 return ERR_PTR(-ENOMEM);
774
775 qp = &sqp->qp;
776
777 err = create_qp_common(dev, pd, init_attr, udata,
778 dev->dev->caps.sqp_start +
779 (init_attr->qp_type == IB_QPT_SMI ? 0 : 2) +
780 init_attr->port_num - 1,
781 qp);
782 if (err) {
783 kfree(sqp);
784 return ERR_PTR(err);
785 }
786
787 qp->port = init_attr->port_num;
788 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
789
790 break;
791 }
792 default:
793
794 return ERR_PTR(-EINVAL);
795 }
796
797 return &qp->ibqp;
798}
799
800int mlx4_ib_destroy_qp(struct ib_qp *qp)
801{
802 struct mlx4_ib_dev *dev = to_mdev(qp->device);
803 struct mlx4_ib_qp *mqp = to_mqp(qp);
804
805 if (is_qp0(dev, mqp))
806 mlx4_CLOSE_PORT(dev->dev, mqp->port);
807
808 destroy_qp_common(dev, mqp, !!qp->pd->uobject);
809
810 if (is_sqp(dev, mqp))
811 kfree(to_msqp(mqp));
812 else
813 kfree(mqp);
814
815 return 0;
816}
817
818static int to_mlx4_st(enum ib_qp_type type)
819{
820 switch (type) {
821 case IB_QPT_RC: return MLX4_QP_ST_RC;
822 case IB_QPT_UC: return MLX4_QP_ST_UC;
823 case IB_QPT_UD: return MLX4_QP_ST_UD;
824 case IB_QPT_SMI:
825 case IB_QPT_GSI: return MLX4_QP_ST_MLX;
826 default: return -1;
827 }
828}
829
830static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
831 int attr_mask)
832{
833 u8 dest_rd_atomic;
834 u32 access_flags;
835 u32 hw_access_flags = 0;
836
837 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
838 dest_rd_atomic = attr->max_dest_rd_atomic;
839 else
840 dest_rd_atomic = qp->resp_depth;
841
842 if (attr_mask & IB_QP_ACCESS_FLAGS)
843 access_flags = attr->qp_access_flags;
844 else
845 access_flags = qp->atomic_rd_en;
846
847 if (!dest_rd_atomic)
848 access_flags &= IB_ACCESS_REMOTE_WRITE;
849
850 if (access_flags & IB_ACCESS_REMOTE_READ)
851 hw_access_flags |= MLX4_QP_BIT_RRE;
852 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
853 hw_access_flags |= MLX4_QP_BIT_RAE;
854 if (access_flags & IB_ACCESS_REMOTE_WRITE)
855 hw_access_flags |= MLX4_QP_BIT_RWE;
856
857 return cpu_to_be32(hw_access_flags);
858}
859
860static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
861 int attr_mask)
862{
863 if (attr_mask & IB_QP_PKEY_INDEX)
864 sqp->pkey_index = attr->pkey_index;
865 if (attr_mask & IB_QP_QKEY)
866 sqp->qkey = attr->qkey;
867 if (attr_mask & IB_QP_SQ_PSN)
868 sqp->send_psn = attr->sq_psn;
869}
870
871static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
872{
873 path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
874}
875
876static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
877 struct mlx4_qp_path *path, u8 port)
878{
879 int err;
880 int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) ==
881 IB_LINK_LAYER_ETHERNET;
882 u8 mac[6];
883 int is_mcast;
884 u16 vlan_tag;
885 int vidx;
886
887 path->grh_mylmc = ah->src_path_bits & 0x7f;
888 path->rlid = cpu_to_be16(ah->dlid);
889 if (ah->static_rate) {
890 path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET;
891 while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
892 !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
893 --path->static_rate;
894 } else
895 path->static_rate = 0;
896 path->counter_index = 0xff;
897
898 if (ah->ah_flags & IB_AH_GRH) {
899 if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
900 printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",
901 ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
902 return -1;
903 }
904
905 path->grh_mylmc |= 1 << 7;
906 path->mgid_index = ah->grh.sgid_index;
907 path->hop_limit = ah->grh.hop_limit;
908 path->tclass_flowlabel =
909 cpu_to_be32((ah->grh.traffic_class << 20) |
910 (ah->grh.flow_label));
911 memcpy(path->rgid, ah->grh.dgid.raw, 16);
912 }
913
914 if (is_eth) {
915 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
916 ((port - 1) << 6) | ((ah->sl & 7) << 3) | ((ah->sl & 8) >> 1);
917
918 if (!(ah->ah_flags & IB_AH_GRH))
919 return -1;
920
921 err = mlx4_ib_resolve_grh(dev, ah, mac, &is_mcast, port);
922 if (err)
923 return err;
924
925 memcpy(path->dmac, mac, 6);
926 path->ackto = MLX4_IB_LINK_TYPE_ETH;
927
928 path->grh_mylmc &= 0x80;
929
930 vlan_tag = rdma_get_vlan_id(&dev->iboe.gid_table[port - 1][ah->grh.sgid_index]);
931 if (vlan_tag < 0x1000) {
932 if (mlx4_find_cached_vlan(dev->dev, port, vlan_tag, &vidx))
933 return -ENOENT;
934
935 path->vlan_index = vidx;
936 path->fl = 1 << 6;
937 }
938 } else
939 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
940 ((port - 1) << 6) | ((ah->sl & 0xf) << 2);
941
942 return 0;
943}
944
945static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
946{
947 struct mlx4_ib_gid_entry *ge, *tmp;
948
949 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
950 if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
951 ge->added = 1;
952 ge->port = qp->port;
953 }
954 }
955}
956
957static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
958 const struct ib_qp_attr *attr, int attr_mask,
959 enum ib_qp_state cur_state, enum ib_qp_state new_state)
960{
961 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
962 struct mlx4_ib_qp *qp = to_mqp(ibqp);
963 struct mlx4_qp_context *context;
964 enum mlx4_qp_optpar optpar = 0;
965 int sqd_event;
966 int err = -EINVAL;
967
968 context = kzalloc(sizeof *context, GFP_KERNEL);
969 if (!context)
970 return -ENOMEM;
971
972 context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
973 (to_mlx4_st(ibqp->qp_type) << 16));
974
975 if (!(attr_mask & IB_QP_PATH_MIG_STATE))
976 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
977 else {
978 optpar |= MLX4_QP_OPTPAR_PM_STATE;
979 switch (attr->path_mig_state) {
980 case IB_MIG_MIGRATED:
981 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
982 break;
983 case IB_MIG_REARM:
984 context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
985 break;
986 case IB_MIG_ARMED:
987 context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
988 break;
989 }
990 }
991
992 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
993 context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
994 else if (ibqp->qp_type == IB_QPT_UD) {
995 if (qp->flags & MLX4_IB_QP_LSO)
996 context->mtu_msgmax = (IB_MTU_4096 << 5) |
997 ilog2(dev->dev->caps.max_gso_sz);
998 else
999 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
1000 } else if (attr_mask & IB_QP_PATH_MTU) {
1001 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
1002 printk(KERN_ERR "path MTU (%u) is invalid\n",
1003 attr->path_mtu);
1004 goto out;
1005 }
1006 context->mtu_msgmax = (attr->path_mtu << 5) |
1007 ilog2(dev->dev->caps.max_msg_sz);
1008 }
1009
1010 if (qp->rq.wqe_cnt)
1011 context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
1012 context->rq_size_stride |= qp->rq.wqe_shift - 4;
1013
1014 if (qp->sq.wqe_cnt)
1015 context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
1016 context->sq_size_stride |= qp->sq.wqe_shift - 4;
1017
1018 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1019 context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
1020
1021 if (qp->ibqp.uobject)
1022 context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index);
1023 else
1024 context->usr_page = cpu_to_be32(dev->priv_uar.index);
1025
1026 if (attr_mask & IB_QP_DEST_QPN)
1027 context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
1028
1029 if (attr_mask & IB_QP_PORT) {
1030 if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
1031 !(attr_mask & IB_QP_AV)) {
1032 mlx4_set_sched(&context->pri_path, attr->port_num);
1033 optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
1034 }
1035 }
1036
1037 if (attr_mask & IB_QP_PKEY_INDEX) {
1038 context->pri_path.pkey_index = attr->pkey_index;
1039 optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
1040 }
1041
1042 if (attr_mask & IB_QP_AV) {
1043 if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,
1044 attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
1045 goto out;
1046
1047 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
1048 MLX4_QP_OPTPAR_SCHED_QUEUE);
1049 }
1050
1051 if (attr_mask & IB_QP_TIMEOUT) {
1052 context->pri_path.ackto |= attr->timeout << 3;
1053 optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
1054 }
1055
1056 if (attr_mask & IB_QP_ALT_PATH) {
1057 if (attr->alt_port_num == 0 ||
1058 attr->alt_port_num > dev->dev->caps.num_ports)
1059 goto out;
1060
1061 if (attr->alt_pkey_index >=
1062 dev->dev->caps.pkey_table_len[attr->alt_port_num])
1063 goto out;
1064
1065 if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
1066 attr->alt_port_num))
1067 goto out;
1068
1069 context->alt_path.pkey_index = attr->alt_pkey_index;
1070 context->alt_path.ackto = attr->alt_timeout << 3;
1071 optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
1072 }
1073
1074 context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn);
1075 context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
1076
1077
1078 if (!qp->ibqp.uobject)
1079 context->params1 |= cpu_to_be32(1 << 11);
1080
1081 if (attr_mask & IB_QP_RNR_RETRY) {
1082 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
1083 optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
1084 }
1085
1086 if (attr_mask & IB_QP_RETRY_CNT) {
1087 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
1088 optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
1089 }
1090
1091 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1092 if (attr->max_rd_atomic)
1093 context->params1 |=
1094 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
1095 optpar |= MLX4_QP_OPTPAR_SRA_MAX;
1096 }
1097
1098 if (attr_mask & IB_QP_SQ_PSN)
1099 context->next_send_psn = cpu_to_be32(attr->sq_psn);
1100
1101 context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn);
1102
1103 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1104 if (attr->max_dest_rd_atomic)
1105 context->params2 |=
1106 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
1107 optpar |= MLX4_QP_OPTPAR_RRA_MAX;
1108 }
1109
1110 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
1111 context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
1112 optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
1113 }
1114
1115 if (ibqp->srq)
1116 context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
1117
1118 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1119 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
1120 optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
1121 }
1122 if (attr_mask & IB_QP_RQ_PSN)
1123 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
1124
1125 context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn);
1126
1127 if (attr_mask & IB_QP_QKEY) {
1128 context->qkey = cpu_to_be32(attr->qkey);
1129 optpar |= MLX4_QP_OPTPAR_Q_KEY;
1130 }
1131
1132 if (ibqp->srq)
1133 context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
1134
1135 if (!ibqp->srq && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1136 context->db_rec_addr = cpu_to_be64(qp->db.dma);
1137
1138 if (cur_state == IB_QPS_INIT &&
1139 new_state == IB_QPS_RTR &&
1140 (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
1141 ibqp->qp_type == IB_QPT_UD)) {
1142 context->pri_path.sched_queue = (qp->port - 1) << 6;
1143 if (is_qp0(dev, qp))
1144 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
1145 else
1146 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
1147 }
1148
1149 if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD &&
1150 attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
1151 sqd_event = 1;
1152 else
1153 sqd_event = 0;
1154
1155 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1156 context->rlkey |= (1 << 4);
1157
1158
1159
1160
1161
1162
1163
1164 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1165 struct mlx4_wqe_ctrl_seg *ctrl;
1166 int i;
1167
1168 for (i = 0; i < qp->sq.wqe_cnt; ++i) {
1169 ctrl = get_send_wqe(qp, i);
1170 ctrl->owner_opcode = cpu_to_be32(1 << 31);
1171 if (qp->sq_max_wqes_per_wr == 1)
1172 ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4);
1173
1174 stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
1175 }
1176 }
1177
1178 err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
1179 to_mlx4_state(new_state), context, optpar,
1180 sqd_event, &qp->mqp);
1181 if (err)
1182 goto out;
1183
1184 qp->state = new_state;
1185
1186 if (attr_mask & IB_QP_ACCESS_FLAGS)
1187 qp->atomic_rd_en = attr->qp_access_flags;
1188 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1189 qp->resp_depth = attr->max_dest_rd_atomic;
1190 if (attr_mask & IB_QP_PORT) {
1191 qp->port = attr->port_num;
1192 update_mcg_macs(dev, qp);
1193 }
1194 if (attr_mask & IB_QP_ALT_PATH)
1195 qp->alt_port = attr->alt_port_num;
1196
1197 if (is_sqp(dev, qp))
1198 store_sqp_attrs(to_msqp(qp), attr, attr_mask);
1199
1200
1201
1202
1203
1204 if (is_qp0(dev, qp)) {
1205 if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
1206 if (mlx4_INIT_PORT(dev->dev, qp->port))
1207 printk(KERN_WARNING "INIT_PORT failed for port %d\n",
1208 qp->port);
1209
1210 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
1211 (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
1212 mlx4_CLOSE_PORT(dev->dev, qp->port);
1213 }
1214
1215
1216
1217
1218
1219 if (new_state == IB_QPS_RESET && !ibqp->uobject) {
1220 mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn,
1221 ibqp->srq ? to_msrq(ibqp->srq): NULL);
1222 if (ibqp->send_cq != ibqp->recv_cq)
1223 mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL);
1224
1225 qp->rq.head = 0;
1226 qp->rq.tail = 0;
1227 qp->sq.head = 0;
1228 qp->sq.tail = 0;
1229 qp->sq_next_wqe = 0;
1230 if (!ibqp->srq)
1231 *qp->db.db = 0;
1232 }
1233
1234out:
1235 kfree(context);
1236 return err;
1237}
1238
1239int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1240 int attr_mask, struct ib_udata *udata)
1241{
1242 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
1243 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1244 enum ib_qp_state cur_state, new_state;
1245 int err = -EINVAL;
1246
1247 mutex_lock(&qp->mutex);
1248
1249 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
1250 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1251
1252 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))
1253 goto out;
1254
1255 if ((attr_mask & IB_QP_PORT) &&
1256 (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) {
1257 goto out;
1258 }
1259
1260 if (attr_mask & IB_QP_PKEY_INDEX) {
1261 int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1262 if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
1263 goto out;
1264 }
1265
1266 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1267 attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
1268 goto out;
1269 }
1270
1271 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1272 attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
1273 goto out;
1274 }
1275
1276 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1277 err = 0;
1278 goto out;
1279 }
1280
1281 err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
1282
1283out:
1284 mutex_unlock(&qp->mutex);
1285 return err;
1286}
1287
1288static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
1289 void *wqe, unsigned *mlx_seg_len)
1290{
1291 struct ib_device *ib_dev = sqp->qp.ibqp.device;
1292 struct mlx4_wqe_mlx_seg *mlx = wqe;
1293 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
1294 struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah);
1295 union ib_gid sgid;
1296 u16 pkey;
1297 int send_size;
1298 int header_size;
1299 int spc;
1300 int i;
1301 int is_eth;
1302 int is_vlan = 0;
1303 int is_grh;
1304 u16 vlan;
1305
1306 send_size = 0;
1307 for (i = 0; i < wr->num_sge; ++i)
1308 send_size += wr->sg_list[i].length;
1309
1310 is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
1311 is_grh = mlx4_ib_ah_grh_present(ah);
1312 if (is_eth) {
1313 ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24,
1314 ah->av.ib.gid_index, &sgid);
1315 vlan = rdma_get_vlan_id(&sgid);
1316 is_vlan = vlan < 0x1000;
1317 }
1318 ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, 0, &sqp->ud_header);
1319
1320 if (!is_eth) {
1321 sqp->ud_header.lrh.service_level =
1322 be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
1323 sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
1324 sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
1325 }
1326
1327 if (is_grh) {
1328 sqp->ud_header.grh.traffic_class =
1329 (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
1330 sqp->ud_header.grh.flow_label =
1331 ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
1332 sqp->ud_header.grh.hop_limit = ah->av.ib.hop_limit;
1333 ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24,
1334 ah->av.ib.gid_index, &sqp->ud_header.grh.source_gid);
1335 memcpy(sqp->ud_header.grh.destination_gid.raw,
1336 ah->av.ib.dgid, 16);
1337 }
1338
1339 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
1340
1341 if (!is_eth) {
1342 mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
1343 (sqp->ud_header.lrh.destination_lid ==
1344 IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
1345 (sqp->ud_header.lrh.service_level << 8));
1346 mlx->rlid = sqp->ud_header.lrh.destination_lid;
1347 }
1348
1349 switch (wr->opcode) {
1350 case IB_WR_SEND:
1351 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
1352 sqp->ud_header.immediate_present = 0;
1353 break;
1354 case IB_WR_SEND_WITH_IMM:
1355 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1356 sqp->ud_header.immediate_present = 1;
1357 sqp->ud_header.immediate_data = wr->ex.imm_data;
1358 break;
1359 default:
1360 return -EINVAL;
1361 }
1362
1363 if (is_eth) {
1364 u8 *smac;
1365
1366 memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
1367
1368 smac = to_mdev(sqp->qp.ibqp.device)->iboe.netdevs[sqp->qp.port - 1]->dev_addr;
1369 memcpy(sqp->ud_header.eth.smac_h, smac, 6);
1370 if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
1371 mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
1372 if (!is_vlan) {
1373 sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
1374 } else {
1375 u16 pcp;
1376
1377 sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
1378 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 27 & 3) << 13;
1379 sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
1380 }
1381 } else {
1382 sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0;
1383 if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
1384 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
1385 }
1386 sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED);
1387 if (!sqp->qp.ibqp.qp_num)
1388 ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
1389 else
1390 ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey);
1391 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
1392 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
1393 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
1394 sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ?
1395 sqp->qkey : wr->wr.ud.remote_qkey);
1396 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
1397
1398 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
1399
1400 if (0) {
1401 printk(KERN_ERR "built UD header of size %d:\n", header_size);
1402 for (i = 0; i < header_size / 4; ++i) {
1403 if (i % 8 == 0)
1404 printk(" [%02x] ", i * 4);
1405 printk(" %08x",
1406 be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
1407 if ((i + 1) % 8 == 0)
1408 printk("\n");
1409 }
1410 printk("\n");
1411 }
1412
1413
1414
1415
1416
1417
1418
1419 spc = MLX4_INLINE_ALIGN -
1420 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
1421 if (header_size <= spc) {
1422 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
1423 memcpy(inl + 1, sqp->header_buf, header_size);
1424 i = 1;
1425 } else {
1426 inl->byte_count = cpu_to_be32(1 << 31 | spc);
1427 memcpy(inl + 1, sqp->header_buf, spc);
1428
1429 inl = (void *) (inl + 1) + spc;
1430 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444 wmb();
1445 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
1446 i = 2;
1447 }
1448
1449 *mlx_seg_len =
1450 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
1451 return 0;
1452}
1453
1454static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
1455{
1456 unsigned cur;
1457 struct mlx4_ib_cq *cq;
1458
1459 cur = wq->head - wq->tail;
1460 if (likely(cur + nreq < wq->max_post))
1461 return 0;
1462
1463 cq = to_mcq(ib_cq);
1464 spin_lock(&cq->lock);
1465 cur = wq->head - wq->tail;
1466 spin_unlock(&cq->lock);
1467
1468 return cur + nreq >= wq->max_post;
1469}
1470
1471static __be32 convert_access(int acc)
1472{
1473 return (acc & IB_ACCESS_REMOTE_ATOMIC ? cpu_to_be32(MLX4_WQE_FMR_PERM_ATOMIC) : 0) |
1474 (acc & IB_ACCESS_REMOTE_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_WRITE) : 0) |
1475 (acc & IB_ACCESS_REMOTE_READ ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_READ) : 0) |
1476 (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) |
1477 cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
1478}
1479
1480static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr)
1481{
1482 struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list);
1483 int i;
1484
1485 for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i)
1486 mfrpl->mapped_page_list[i] =
1487 cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] |
1488 MLX4_MTT_FLAG_PRESENT);
1489
1490 fseg->flags = convert_access(wr->wr.fast_reg.access_flags);
1491 fseg->mem_key = cpu_to_be32(wr->wr.fast_reg.rkey);
1492 fseg->buf_list = cpu_to_be64(mfrpl->map);
1493 fseg->start_addr = cpu_to_be64(wr->wr.fast_reg.iova_start);
1494 fseg->reg_len = cpu_to_be64(wr->wr.fast_reg.length);
1495 fseg->offset = 0;
1496 fseg->page_size = cpu_to_be32(wr->wr.fast_reg.page_shift);
1497 fseg->reserved[0] = 0;
1498 fseg->reserved[1] = 0;
1499}
1500
1501static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
1502{
1503 iseg->flags = 0;
1504 iseg->mem_key = cpu_to_be32(rkey);
1505 iseg->guest_id = 0;
1506 iseg->pa = 0;
1507}
1508
1509static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
1510 u64 remote_addr, u32 rkey)
1511{
1512 rseg->raddr = cpu_to_be64(remote_addr);
1513 rseg->rkey = cpu_to_be32(rkey);
1514 rseg->reserved = 0;
1515}
1516
1517static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, struct ib_send_wr *wr)
1518{
1519 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
1520 aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap);
1521 aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add);
1522 } else if (wr->opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
1523 aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
1524 aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add_mask);
1525 } else {
1526 aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
1527 aseg->compare = 0;
1528 }
1529
1530}
1531
1532static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
1533 struct ib_send_wr *wr)
1534{
1535 aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap);
1536 aseg->swap_add_mask = cpu_to_be64(wr->wr.atomic.swap_mask);
1537 aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add);
1538 aseg->compare_mask = cpu_to_be64(wr->wr.atomic.compare_add_mask);
1539}
1540
1541static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
1542 struct ib_send_wr *wr, __be16 *vlan)
1543{
1544 memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
1545 dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn);
1546 dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey);
1547 dseg->vlan = to_mah(wr->wr.ud.ah)->av.eth.vlan;
1548 memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->av.eth.mac, 6);
1549 *vlan = dseg->vlan;
1550}
1551
1552static void set_mlx_icrc_seg(void *dseg)
1553{
1554 u32 *t = dseg;
1555 struct mlx4_wqe_inline_seg *iseg = dseg;
1556
1557 t[1] = 0;
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567 wmb();
1568
1569 iseg->byte_count = cpu_to_be32((1 << 31) | 4);
1570}
1571
1572static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
1573{
1574 dseg->lkey = cpu_to_be32(sg->lkey);
1575 dseg->addr = cpu_to_be64(sg->addr);
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585 wmb();
1586
1587 dseg->byte_count = cpu_to_be32(sg->length);
1588}
1589
1590static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
1591{
1592 dseg->byte_count = cpu_to_be32(sg->length);
1593 dseg->lkey = cpu_to_be32(sg->lkey);
1594 dseg->addr = cpu_to_be64(sg->addr);
1595}
1596
1597static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr,
1598 struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
1599 __be32 *lso_hdr_sz, __be32 *blh)
1600{
1601 unsigned halign = ALIGN(sizeof *wqe + wr->wr.ud.hlen, 16);
1602
1603 if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
1604 *blh = cpu_to_be32(1 << 6);
1605
1606 if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
1607 wr->num_sge > qp->sq.max_gs - (halign >> 4)))
1608 return -EINVAL;
1609
1610 memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen);
1611
1612 *lso_hdr_sz = cpu_to_be32((wr->wr.ud.mss - wr->wr.ud.hlen) << 16 |
1613 wr->wr.ud.hlen);
1614 *lso_seg_len = halign;
1615 return 0;
1616}
1617
1618static __be32 send_ieth(struct ib_send_wr *wr)
1619{
1620 switch (wr->opcode) {
1621 case IB_WR_SEND_WITH_IMM:
1622 case IB_WR_RDMA_WRITE_WITH_IMM:
1623 return wr->ex.imm_data;
1624
1625 case IB_WR_SEND_WITH_INV:
1626 return cpu_to_be32(wr->ex.invalidate_rkey);
1627
1628 default:
1629 return 0;
1630 }
1631}
1632
1633int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1634 struct ib_send_wr **bad_wr)
1635{
1636 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1637 void *wqe;
1638 struct mlx4_wqe_ctrl_seg *ctrl;
1639 struct mlx4_wqe_data_seg *dseg;
1640 unsigned long flags;
1641 int nreq;
1642 int err = 0;
1643 unsigned ind;
1644 int uninitialized_var(stamp);
1645 int uninitialized_var(size);
1646 unsigned uninitialized_var(seglen);
1647 __be32 dummy;
1648 __be32 *lso_wqe;
1649 __be32 uninitialized_var(lso_hdr_sz);
1650 __be32 blh;
1651 int i;
1652 __be16 vlan = cpu_to_be16(0xffff);
1653
1654 spin_lock_irqsave(&qp->sq.lock, flags);
1655
1656 ind = qp->sq_next_wqe;
1657
1658 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1659 lso_wqe = &dummy;
1660 blh = 0;
1661
1662 if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
1663 err = -ENOMEM;
1664 *bad_wr = wr;
1665 goto out;
1666 }
1667
1668 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
1669 err = -EINVAL;
1670 *bad_wr = wr;
1671 goto out;
1672 }
1673
1674 ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
1675 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
1676
1677 ctrl->srcrb_flags =
1678 (wr->send_flags & IB_SEND_SIGNALED ?
1679 cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
1680 (wr->send_flags & IB_SEND_SOLICITED ?
1681 cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
1682 ((wr->send_flags & IB_SEND_IP_CSUM) ?
1683 cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
1684 MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
1685 qp->sq_signal_bits;
1686
1687 ctrl->imm = send_ieth(wr);
1688
1689 wqe += sizeof *ctrl;
1690 size = sizeof *ctrl / 16;
1691
1692 switch (ibqp->qp_type) {
1693 case IB_QPT_RC:
1694 case IB_QPT_UC:
1695 switch (wr->opcode) {
1696 case IB_WR_ATOMIC_CMP_AND_SWP:
1697 case IB_WR_ATOMIC_FETCH_AND_ADD:
1698 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
1699 set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
1700 wr->wr.atomic.rkey);
1701 wqe += sizeof (struct mlx4_wqe_raddr_seg);
1702
1703 set_atomic_seg(wqe, wr);
1704 wqe += sizeof (struct mlx4_wqe_atomic_seg);
1705
1706 size += (sizeof (struct mlx4_wqe_raddr_seg) +
1707 sizeof (struct mlx4_wqe_atomic_seg)) / 16;
1708
1709 break;
1710
1711 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
1712 set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
1713 wr->wr.atomic.rkey);
1714 wqe += sizeof (struct mlx4_wqe_raddr_seg);
1715
1716 set_masked_atomic_seg(wqe, wr);
1717 wqe += sizeof (struct mlx4_wqe_masked_atomic_seg);
1718
1719 size += (sizeof (struct mlx4_wqe_raddr_seg) +
1720 sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
1721
1722 break;
1723
1724 case IB_WR_RDMA_READ:
1725 case IB_WR_RDMA_WRITE:
1726 case IB_WR_RDMA_WRITE_WITH_IMM:
1727 set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
1728 wr->wr.rdma.rkey);
1729 wqe += sizeof (struct mlx4_wqe_raddr_seg);
1730 size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
1731 break;
1732
1733 case IB_WR_LOCAL_INV:
1734 ctrl->srcrb_flags |=
1735 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
1736 set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
1737 wqe += sizeof (struct mlx4_wqe_local_inval_seg);
1738 size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
1739 break;
1740
1741 case IB_WR_FAST_REG_MR:
1742 ctrl->srcrb_flags |=
1743 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
1744 set_fmr_seg(wqe, wr);
1745 wqe += sizeof (struct mlx4_wqe_fmr_seg);
1746 size += sizeof (struct mlx4_wqe_fmr_seg) / 16;
1747 break;
1748
1749 default:
1750
1751 break;
1752 }
1753 break;
1754
1755 case IB_QPT_UD:
1756 set_datagram_seg(wqe, wr, &vlan);
1757 wqe += sizeof (struct mlx4_wqe_datagram_seg);
1758 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
1759
1760 if (wr->opcode == IB_WR_LSO) {
1761 err = build_lso_seg(wqe, wr, qp, &seglen, &lso_hdr_sz, &blh);
1762 if (unlikely(err)) {
1763 *bad_wr = wr;
1764 goto out;
1765 }
1766 lso_wqe = (__be32 *) wqe;
1767 wqe += seglen;
1768 size += seglen / 16;
1769 }
1770 break;
1771
1772 case IB_QPT_SMI:
1773 case IB_QPT_GSI:
1774 err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen);
1775 if (unlikely(err)) {
1776 *bad_wr = wr;
1777 goto out;
1778 }
1779 wqe += seglen;
1780 size += seglen / 16;
1781 break;
1782
1783 default:
1784 break;
1785 }
1786
1787
1788
1789
1790
1791
1792
1793
1794 dseg = wqe;
1795 dseg += wr->num_sge - 1;
1796 size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
1797
1798
1799 if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI ||
1800 qp->ibqp.qp_type == IB_QPT_GSI)) {
1801 set_mlx_icrc_seg(dseg + 1);
1802 size += sizeof (struct mlx4_wqe_data_seg) / 16;
1803 }
1804
1805 for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
1806 set_data_seg(dseg, wr->sg_list + i);
1807
1808
1809
1810
1811
1812
1813 wmb();
1814 *lso_wqe = lso_hdr_sz;
1815
1816 ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ?
1817 MLX4_WQE_CTRL_FENCE : 0) | size;
1818
1819 if (be16_to_cpu(vlan) < 0x1000) {
1820 ctrl->ins_vlan = 1 << 6;
1821 ctrl->vlan_tag = vlan;
1822 }
1823
1824
1825
1826
1827
1828
1829 wmb();
1830
1831 if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
1832 err = -EINVAL;
1833 goto out;
1834 }
1835
1836 ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
1837 (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
1838
1839 stamp = ind + qp->sq_spare_wqes;
1840 ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851 if (wr->next) {
1852 stamp_send_wqe(qp, stamp, size * 16);
1853 ind = pad_wraparound(qp, ind);
1854 }
1855 }
1856
1857out:
1858 if (likely(nreq)) {
1859 qp->sq.head += nreq;
1860
1861
1862
1863
1864
1865 wmb();
1866
1867 writel(qp->doorbell_qpn,
1868 to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
1869
1870
1871
1872
1873
1874 mmiowb();
1875
1876 stamp_send_wqe(qp, stamp, size * 16);
1877
1878 ind = pad_wraparound(qp, ind);
1879 qp->sq_next_wqe = ind;
1880 }
1881
1882 spin_unlock_irqrestore(&qp->sq.lock, flags);
1883
1884 return err;
1885}
1886
1887int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
1888 struct ib_recv_wr **bad_wr)
1889{
1890 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1891 struct mlx4_wqe_data_seg *scat;
1892 unsigned long flags;
1893 int err = 0;
1894 int nreq;
1895 int ind;
1896 int i;
1897
1898 spin_lock_irqsave(&qp->rq.lock, flags);
1899
1900 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1901
1902 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1903 if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1904 err = -ENOMEM;
1905 *bad_wr = wr;
1906 goto out;
1907 }
1908
1909 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1910 err = -EINVAL;
1911 *bad_wr = wr;
1912 goto out;
1913 }
1914
1915 scat = get_recv_wqe(qp, ind);
1916
1917 for (i = 0; i < wr->num_sge; ++i)
1918 __set_data_seg(scat + i, wr->sg_list + i);
1919
1920 if (i < qp->rq.max_gs) {
1921 scat[i].byte_count = 0;
1922 scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
1923 scat[i].addr = 0;
1924 }
1925
1926 qp->rq.wrid[ind] = wr->wr_id;
1927
1928 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1929 }
1930
1931out:
1932 if (likely(nreq)) {
1933 qp->rq.head += nreq;
1934
1935
1936
1937
1938
1939 wmb();
1940
1941 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1942 }
1943
1944 spin_unlock_irqrestore(&qp->rq.lock, flags);
1945
1946 return err;
1947}
1948
1949static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
1950{
1951 switch (mlx4_state) {
1952 case MLX4_QP_STATE_RST: return IB_QPS_RESET;
1953 case MLX4_QP_STATE_INIT: return IB_QPS_INIT;
1954 case MLX4_QP_STATE_RTR: return IB_QPS_RTR;
1955 case MLX4_QP_STATE_RTS: return IB_QPS_RTS;
1956 case MLX4_QP_STATE_SQ_DRAINING:
1957 case MLX4_QP_STATE_SQD: return IB_QPS_SQD;
1958 case MLX4_QP_STATE_SQER: return IB_QPS_SQE;
1959 case MLX4_QP_STATE_ERR: return IB_QPS_ERR;
1960 default: return -1;
1961 }
1962}
1963
1964static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
1965{
1966 switch (mlx4_mig_state) {
1967 case MLX4_QP_PM_ARMED: return IB_MIG_ARMED;
1968 case MLX4_QP_PM_REARM: return IB_MIG_REARM;
1969 case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
1970 default: return -1;
1971 }
1972}
1973
1974static int to_ib_qp_access_flags(int mlx4_flags)
1975{
1976 int ib_flags = 0;
1977
1978 if (mlx4_flags & MLX4_QP_BIT_RRE)
1979 ib_flags |= IB_ACCESS_REMOTE_READ;
1980 if (mlx4_flags & MLX4_QP_BIT_RWE)
1981 ib_flags |= IB_ACCESS_REMOTE_WRITE;
1982 if (mlx4_flags & MLX4_QP_BIT_RAE)
1983 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
1984
1985 return ib_flags;
1986}
1987
1988static void to_ib_ah_attr(struct mlx4_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr,
1989 struct mlx4_qp_path *path)
1990{
1991 struct mlx4_dev *dev = ibdev->dev;
1992 int is_eth;
1993
1994 memset(ib_ah_attr, 0, sizeof *ib_ah_attr);
1995 ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1;
1996
1997 if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports)
1998 return;
1999
2000 is_eth = rdma_port_get_link_layer(&ibdev->ib_dev, ib_ah_attr->port_num) ==
2001 IB_LINK_LAYER_ETHERNET;
2002 if (is_eth)
2003 ib_ah_attr->sl = ((path->sched_queue >> 3) & 0x7) |
2004 ((path->sched_queue & 4) << 1);
2005 else
2006 ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf;
2007
2008 ib_ah_attr->dlid = be16_to_cpu(path->rlid);
2009 ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f;
2010 ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0;
2011 ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
2012 if (ib_ah_attr->ah_flags) {
2013 ib_ah_attr->grh.sgid_index = path->mgid_index;
2014 ib_ah_attr->grh.hop_limit = path->hop_limit;
2015 ib_ah_attr->grh.traffic_class =
2016 (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff;
2017 ib_ah_attr->grh.flow_label =
2018 be32_to_cpu(path->tclass_flowlabel) & 0xfffff;
2019 memcpy(ib_ah_attr->grh.dgid.raw,
2020 path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
2021 }
2022}
2023
2024int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
2025 struct ib_qp_init_attr *qp_init_attr)
2026{
2027 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2028 struct mlx4_ib_qp *qp = to_mqp(ibqp);
2029 struct mlx4_qp_context context;
2030 int mlx4_state;
2031 int err = 0;
2032
2033 mutex_lock(&qp->mutex);
2034
2035 if (qp->state == IB_QPS_RESET) {
2036 qp_attr->qp_state = IB_QPS_RESET;
2037 goto done;
2038 }
2039
2040 err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
2041 if (err) {
2042 err = -EINVAL;
2043 goto out;
2044 }
2045
2046 mlx4_state = be32_to_cpu(context.flags) >> 28;
2047
2048 qp->state = to_ib_qp_state(mlx4_state);
2049 qp_attr->qp_state = qp->state;
2050 qp_attr->path_mtu = context.mtu_msgmax >> 5;
2051 qp_attr->path_mig_state =
2052 to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
2053 qp_attr->qkey = be32_to_cpu(context.qkey);
2054 qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
2055 qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff;
2056 qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff;
2057 qp_attr->qp_access_flags =
2058 to_ib_qp_access_flags(be32_to_cpu(context.params2));
2059
2060 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
2061 to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
2062 to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
2063 qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
2064 qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num;
2065 }
2066
2067 qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
2068 if (qp_attr->qp_state == IB_QPS_INIT)
2069 qp_attr->port_num = qp->port;
2070 else
2071 qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
2072
2073
2074 qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
2075
2076 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
2077
2078 qp_attr->max_dest_rd_atomic =
2079 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
2080 qp_attr->min_rnr_timer =
2081 (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
2082 qp_attr->timeout = context.pri_path.ackto >> 3;
2083 qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7;
2084 qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7;
2085 qp_attr->alt_timeout = context.alt_path.ackto >> 3;
2086
2087done:
2088 qp_attr->cur_qp_state = qp_attr->qp_state;
2089 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
2090 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
2091
2092 if (!ibqp->uobject) {
2093 qp_attr->cap.max_send_wr = qp->sq.wqe_cnt;
2094 qp_attr->cap.max_send_sge = qp->sq.max_gs;
2095 } else {
2096 qp_attr->cap.max_send_wr = 0;
2097 qp_attr->cap.max_send_sge = 0;
2098 }
2099
2100
2101
2102
2103
2104 qp_attr->cap.max_inline_data = 0;
2105
2106 qp_init_attr->cap = qp_attr->cap;
2107
2108 qp_init_attr->create_flags = 0;
2109 if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
2110 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
2111
2112 if (qp->flags & MLX4_IB_QP_LSO)
2113 qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
2114
2115out:
2116 mutex_unlock(&qp->mutex);
2117 return err;
2118}
2119
2120