1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/log2.h>
35
36#include <rdma/ib_cache.h>
37#include <rdma/ib_pack.h>
38
39#include <linux/mlx4/qp.h>
40
41#include "mlx4_ib.h"
42#include "user.h"
43
44enum {
45 MLX4_IB_ACK_REQ_FREQ = 8,
46};
47
48enum {
49 MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,
50 MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f
51};
52
53enum {
54
55
56
57 MLX4_IB_UD_HEADER_SIZE = 72
58};
59
60struct mlx4_ib_sqp {
61 struct mlx4_ib_qp qp;
62 int pkey_index;
63 u32 qkey;
64 u32 send_psn;
65 struct ib_ud_header ud_header;
66 u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
67};
68
69enum {
70 MLX4_IB_MIN_SQ_STRIDE = 6
71};
72
73static const __be32 mlx4_ib_opcode[] = {
74 [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND),
75 [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO),
76 [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM),
77 [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
78 [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
79 [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ),
80 [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
81 [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
82 [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
83 [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
84 [IB_WR_FAST_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR),
85};
86
87static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
88{
89 return container_of(mqp, struct mlx4_ib_sqp, qp);
90}
91
92static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
93{
94 return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
95 qp->mqp.qpn <= dev->dev->caps.sqp_start + 3;
96}
97
98static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
99{
100 return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
101 qp->mqp.qpn <= dev->dev->caps.sqp_start + 1;
102}
103
104static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
105{
106 return mlx4_buf_offset(&qp->buf, offset);
107}
108
109static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
110{
111 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
112}
113
114static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
115{
116 return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
117}
118
119
120
121
122
123
124
125
126
127
128static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
129{
130 __be32 *wqe;
131 int i;
132 int s;
133 int ind;
134 void *buf;
135 __be32 stamp;
136 struct mlx4_wqe_ctrl_seg *ctrl;
137
138 if (qp->sq_max_wqes_per_wr > 1) {
139 s = roundup(size, 1U << qp->sq.wqe_shift);
140 for (i = 0; i < s; i += 64) {
141 ind = (i >> qp->sq.wqe_shift) + n;
142 stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
143 cpu_to_be32(0xffffffff);
144 buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
145 wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
146 *wqe = stamp;
147 }
148 } else {
149 ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
150 s = (ctrl->fence_size & 0x3f) << 4;
151 for (i = 64; i < s; i += 64) {
152 wqe = buf + i;
153 *wqe = cpu_to_be32(0xffffffff);
154 }
155 }
156}
157
158static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
159{
160 struct mlx4_wqe_ctrl_seg *ctrl;
161 struct mlx4_wqe_inline_seg *inl;
162 void *wqe;
163 int s;
164
165 ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
166 s = sizeof(struct mlx4_wqe_ctrl_seg);
167
168 if (qp->ibqp.qp_type == IB_QPT_UD) {
169 struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
170 struct mlx4_av *av = (struct mlx4_av *)dgram->av;
171 memset(dgram, 0, sizeof *dgram);
172 av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
173 s += sizeof(struct mlx4_wqe_datagram_seg);
174 }
175
176
177 if (size > s) {
178 inl = wqe + s;
179 inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
180 }
181 ctrl->srcrb_flags = 0;
182 ctrl->fence_size = size / 16;
183
184
185
186
187 wmb();
188
189 ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
190 (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
191
192 stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
193}
194
195
196static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
197{
198 unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
199 if (unlikely(s < qp->sq_max_wqes_per_wr)) {
200 post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
201 ind += s;
202 }
203 return ind;
204}
205
206static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
207{
208 struct ib_event event;
209 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
210
211 if (type == MLX4_EVENT_TYPE_PATH_MIG)
212 to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
213
214 if (ibqp->event_handler) {
215 event.device = ibqp->device;
216 event.element.qp = ibqp;
217 switch (type) {
218 case MLX4_EVENT_TYPE_PATH_MIG:
219 event.event = IB_EVENT_PATH_MIG;
220 break;
221 case MLX4_EVENT_TYPE_COMM_EST:
222 event.event = IB_EVENT_COMM_EST;
223 break;
224 case MLX4_EVENT_TYPE_SQ_DRAINED:
225 event.event = IB_EVENT_SQ_DRAINED;
226 break;
227 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
228 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
229 break;
230 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
231 event.event = IB_EVENT_QP_FATAL;
232 break;
233 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
234 event.event = IB_EVENT_PATH_MIG_ERR;
235 break;
236 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
237 event.event = IB_EVENT_QP_REQ_ERR;
238 break;
239 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
240 event.event = IB_EVENT_QP_ACCESS_ERR;
241 break;
242 default:
243 printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
244 "on QP %06x\n", type, qp->qpn);
245 return;
246 }
247
248 ibqp->event_handler(&event, ibqp->qp_context);
249 }
250}
251
252static int send_wqe_overhead(enum ib_qp_type type, u32 flags)
253{
254
255
256
257
258
259
260 switch (type) {
261 case IB_QPT_UD:
262 return sizeof (struct mlx4_wqe_ctrl_seg) +
263 sizeof (struct mlx4_wqe_datagram_seg) +
264 ((flags & MLX4_IB_QP_LSO) ? 64 : 0);
265 case IB_QPT_UC:
266 return sizeof (struct mlx4_wqe_ctrl_seg) +
267 sizeof (struct mlx4_wqe_raddr_seg);
268 case IB_QPT_RC:
269 return sizeof (struct mlx4_wqe_ctrl_seg) +
270 sizeof (struct mlx4_wqe_atomic_seg) +
271 sizeof (struct mlx4_wqe_raddr_seg);
272 case IB_QPT_SMI:
273 case IB_QPT_GSI:
274 return sizeof (struct mlx4_wqe_ctrl_seg) +
275 ALIGN(MLX4_IB_UD_HEADER_SIZE +
276 DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
277 MLX4_INLINE_ALIGN) *
278 sizeof (struct mlx4_wqe_inline_seg),
279 sizeof (struct mlx4_wqe_data_seg)) +
280 ALIGN(4 +
281 sizeof (struct mlx4_wqe_inline_seg),
282 sizeof (struct mlx4_wqe_data_seg));
283 default:
284 return sizeof (struct mlx4_wqe_ctrl_seg);
285 }
286}
287
288static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
289 int is_user, int has_srq, struct mlx4_ib_qp *qp)
290{
291
292 if (cap->max_recv_wr > dev->dev->caps.max_wqes ||
293 cap->max_recv_sge > dev->dev->caps.max_rq_sg)
294 return -EINVAL;
295
296 if (has_srq) {
297
298 if (cap->max_recv_wr)
299 return -EINVAL;
300
301 qp->rq.wqe_cnt = qp->rq.max_gs = 0;
302 } else {
303
304 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
305 return -EINVAL;
306
307 qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));
308 qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));
309 qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
310 }
311
312 cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
313 cap->max_recv_sge = qp->rq.max_gs;
314
315 return 0;
316}
317
318static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
319 enum ib_qp_type type, struct mlx4_ib_qp *qp)
320{
321 int s;
322
323
324 if (cap->max_send_wr > dev->dev->caps.max_wqes ||
325 cap->max_send_sge > dev->dev->caps.max_sq_sg ||
326 cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
327 sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
328 return -EINVAL;
329
330
331
332
333
334 if ((type == IB_QPT_SMI || type == IB_QPT_GSI) &&
335 cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
336 return -EINVAL;
337
338 s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
339 cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
340 send_wqe_overhead(type, qp->flags);
341
342 if (s > dev->dev->caps.max_sq_desc_sz)
343 return -EINVAL;
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376 if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
377 qp->sq_signal_bits && BITS_PER_LONG == 64 &&
378 type != IB_QPT_SMI && type != IB_QPT_GSI)
379 qp->sq.wqe_shift = ilog2(64);
380 else
381 qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
382
383 for (;;) {
384 qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
385
386
387
388
389
390 qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
391 qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
392 qp->sq_max_wqes_per_wr +
393 qp->sq_spare_wqes);
394
395 if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
396 break;
397
398 if (qp->sq_max_wqes_per_wr <= 1)
399 return -EINVAL;
400
401 ++qp->sq.wqe_shift;
402 }
403
404 qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
405 (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
406 send_wqe_overhead(type, qp->flags)) /
407 sizeof (struct mlx4_wqe_data_seg);
408
409 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
410 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
411 if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
412 qp->rq.offset = 0;
413 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
414 } else {
415 qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
416 qp->sq.offset = 0;
417 }
418
419 cap->max_send_wr = qp->sq.max_post =
420 (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
421 cap->max_send_sge = min(qp->sq.max_gs,
422 min(dev->dev->caps.max_sq_sg,
423 dev->dev->caps.max_rq_sg));
424
425 cap->max_inline_data = 0;
426
427 return 0;
428}
429
430static int set_user_sq_size(struct mlx4_ib_dev *dev,
431 struct mlx4_ib_qp *qp,
432 struct mlx4_ib_create_qp *ucmd)
433{
434
435 if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||
436 ucmd->log_sq_stride >
437 ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
438 ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
439 return -EINVAL;
440
441 qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
442 qp->sq.wqe_shift = ucmd->log_sq_stride;
443
444 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
445 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
446
447 return 0;
448}
449
450static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
451 struct ib_qp_init_attr *init_attr,
452 struct ib_udata *udata, int sqpn, struct mlx4_ib_qp *qp)
453{
454 int qpn;
455 int err;
456
457 mutex_init(&qp->mutex);
458 spin_lock_init(&qp->sq.lock);
459 spin_lock_init(&qp->rq.lock);
460
461 qp->state = IB_QPS_RESET;
462 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
463 qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
464
465 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, !!init_attr->srq, qp);
466 if (err)
467 goto err;
468
469 if (pd->uobject) {
470 struct mlx4_ib_create_qp ucmd;
471
472 if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
473 err = -EFAULT;
474 goto err;
475 }
476
477 qp->sq_no_prefetch = ucmd.sq_no_prefetch;
478
479 err = set_user_sq_size(dev, qp, &ucmd);
480 if (err)
481 goto err;
482
483 qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
484 qp->buf_size, 0, 0);
485 if (IS_ERR(qp->umem)) {
486 err = PTR_ERR(qp->umem);
487 goto err;
488 }
489
490 err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem),
491 ilog2(qp->umem->page_size), &qp->mtt);
492 if (err)
493 goto err_buf;
494
495 err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
496 if (err)
497 goto err_mtt;
498
499 if (!init_attr->srq) {
500 err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
501 ucmd.db_addr, &qp->db);
502 if (err)
503 goto err_mtt;
504 }
505 } else {
506 qp->sq_no_prefetch = 0;
507
508 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
509 qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
510
511 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
512 qp->flags |= MLX4_IB_QP_LSO;
513
514 err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp);
515 if (err)
516 goto err;
517
518 if (!init_attr->srq) {
519 err = mlx4_db_alloc(dev->dev, &qp->db, 0);
520 if (err)
521 goto err;
522
523 *qp->db.db = 0;
524 }
525
526 if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) {
527 err = -ENOMEM;
528 goto err_db;
529 }
530
531 err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
532 &qp->mtt);
533 if (err)
534 goto err_buf;
535
536 err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
537 if (err)
538 goto err_mtt;
539
540 qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL);
541 qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL);
542
543 if (!qp->sq.wrid || !qp->rq.wrid) {
544 err = -ENOMEM;
545 goto err_wrid;
546 }
547 }
548
549 if (sqpn) {
550 qpn = sqpn;
551 } else {
552 err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn);
553 if (err)
554 goto err_wrid;
555 }
556
557 err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
558 if (err)
559 goto err_qpn;
560
561
562
563
564
565
566 qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
567
568 qp->mqp.event = mlx4_ib_qp_event;
569
570 return 0;
571
572err_qpn:
573 if (!sqpn)
574 mlx4_qp_release_range(dev->dev, qpn, 1);
575
576err_wrid:
577 if (pd->uobject) {
578 if (!init_attr->srq)
579 mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context),
580 &qp->db);
581 } else {
582 kfree(qp->sq.wrid);
583 kfree(qp->rq.wrid);
584 }
585
586err_mtt:
587 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
588
589err_buf:
590 if (pd->uobject)
591 ib_umem_release(qp->umem);
592 else
593 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
594
595err_db:
596 if (!pd->uobject && !init_attr->srq)
597 mlx4_db_free(dev->dev, &qp->db);
598
599err:
600 return err;
601}
602
603static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
604{
605 switch (state) {
606 case IB_QPS_RESET: return MLX4_QP_STATE_RST;
607 case IB_QPS_INIT: return MLX4_QP_STATE_INIT;
608 case IB_QPS_RTR: return MLX4_QP_STATE_RTR;
609 case IB_QPS_RTS: return MLX4_QP_STATE_RTS;
610 case IB_QPS_SQD: return MLX4_QP_STATE_SQD;
611 case IB_QPS_SQE: return MLX4_QP_STATE_SQER;
612 case IB_QPS_ERR: return MLX4_QP_STATE_ERR;
613 default: return -1;
614 }
615}
616
617static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
618 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
619{
620 if (send_cq == recv_cq) {
621 spin_lock_irq(&send_cq->lock);
622 __acquire(&recv_cq->lock);
623 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
624 spin_lock_irq(&send_cq->lock);
625 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
626 } else {
627 spin_lock_irq(&recv_cq->lock);
628 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
629 }
630}
631
632static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
633 __releases(&send_cq->lock) __releases(&recv_cq->lock)
634{
635 if (send_cq == recv_cq) {
636 __release(&recv_cq->lock);
637 spin_unlock_irq(&send_cq->lock);
638 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
639 spin_unlock(&recv_cq->lock);
640 spin_unlock_irq(&send_cq->lock);
641 } else {
642 spin_unlock(&send_cq->lock);
643 spin_unlock_irq(&recv_cq->lock);
644 }
645}
646
647static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
648 int is_user)
649{
650 struct mlx4_ib_cq *send_cq, *recv_cq;
651
652 if (qp->state != IB_QPS_RESET)
653 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
654 MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
655 printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n",
656 qp->mqp.qpn);
657
658 send_cq = to_mcq(qp->ibqp.send_cq);
659 recv_cq = to_mcq(qp->ibqp.recv_cq);
660
661 mlx4_ib_lock_cqs(send_cq, recv_cq);
662
663 if (!is_user) {
664 __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
665 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
666 if (send_cq != recv_cq)
667 __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
668 }
669
670 mlx4_qp_remove(dev->dev, &qp->mqp);
671
672 mlx4_ib_unlock_cqs(send_cq, recv_cq);
673
674 mlx4_qp_free(dev->dev, &qp->mqp);
675
676 if (!is_sqp(dev, qp))
677 mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
678
679 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
680
681 if (is_user) {
682 if (!qp->ibqp.srq)
683 mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
684 &qp->db);
685 ib_umem_release(qp->umem);
686 } else {
687 kfree(qp->sq.wrid);
688 kfree(qp->rq.wrid);
689 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
690 if (!qp->ibqp.srq)
691 mlx4_db_free(dev->dev, &qp->db);
692 }
693}
694
695struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
696 struct ib_qp_init_attr *init_attr,
697 struct ib_udata *udata)
698{
699 struct mlx4_ib_dev *dev = to_mdev(pd->device);
700 struct mlx4_ib_sqp *sqp;
701 struct mlx4_ib_qp *qp;
702 int err;
703
704
705
706
707
708 if (init_attr->create_flags & ~(IB_QP_CREATE_IPOIB_UD_LSO |
709 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK))
710 return ERR_PTR(-EINVAL);
711
712 if (init_attr->create_flags &&
713 (pd->uobject || init_attr->qp_type != IB_QPT_UD))
714 return ERR_PTR(-EINVAL);
715
716 switch (init_attr->qp_type) {
717 case IB_QPT_RC:
718 case IB_QPT_UC:
719 case IB_QPT_UD:
720 {
721 qp = kzalloc(sizeof *qp, GFP_KERNEL);
722 if (!qp)
723 return ERR_PTR(-ENOMEM);
724
725 err = create_qp_common(dev, pd, init_attr, udata, 0, qp);
726 if (err) {
727 kfree(qp);
728 return ERR_PTR(err);
729 }
730
731 qp->ibqp.qp_num = qp->mqp.qpn;
732
733 break;
734 }
735 case IB_QPT_SMI:
736 case IB_QPT_GSI:
737 {
738
739 if (pd->uobject)
740 return ERR_PTR(-EINVAL);
741
742 sqp = kzalloc(sizeof *sqp, GFP_KERNEL);
743 if (!sqp)
744 return ERR_PTR(-ENOMEM);
745
746 qp = &sqp->qp;
747
748 err = create_qp_common(dev, pd, init_attr, udata,
749 dev->dev->caps.sqp_start +
750 (init_attr->qp_type == IB_QPT_SMI ? 0 : 2) +
751 init_attr->port_num - 1,
752 qp);
753 if (err) {
754 kfree(sqp);
755 return ERR_PTR(err);
756 }
757
758 qp->port = init_attr->port_num;
759 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
760
761 break;
762 }
763 default:
764
765 return ERR_PTR(-EINVAL);
766 }
767
768 return &qp->ibqp;
769}
770
771int mlx4_ib_destroy_qp(struct ib_qp *qp)
772{
773 struct mlx4_ib_dev *dev = to_mdev(qp->device);
774 struct mlx4_ib_qp *mqp = to_mqp(qp);
775
776 if (is_qp0(dev, mqp))
777 mlx4_CLOSE_PORT(dev->dev, mqp->port);
778
779 destroy_qp_common(dev, mqp, !!qp->pd->uobject);
780
781 if (is_sqp(dev, mqp))
782 kfree(to_msqp(mqp));
783 else
784 kfree(mqp);
785
786 return 0;
787}
788
789static int to_mlx4_st(enum ib_qp_type type)
790{
791 switch (type) {
792 case IB_QPT_RC: return MLX4_QP_ST_RC;
793 case IB_QPT_UC: return MLX4_QP_ST_UC;
794 case IB_QPT_UD: return MLX4_QP_ST_UD;
795 case IB_QPT_SMI:
796 case IB_QPT_GSI: return MLX4_QP_ST_MLX;
797 default: return -1;
798 }
799}
800
801static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
802 int attr_mask)
803{
804 u8 dest_rd_atomic;
805 u32 access_flags;
806 u32 hw_access_flags = 0;
807
808 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
809 dest_rd_atomic = attr->max_dest_rd_atomic;
810 else
811 dest_rd_atomic = qp->resp_depth;
812
813 if (attr_mask & IB_QP_ACCESS_FLAGS)
814 access_flags = attr->qp_access_flags;
815 else
816 access_flags = qp->atomic_rd_en;
817
818 if (!dest_rd_atomic)
819 access_flags &= IB_ACCESS_REMOTE_WRITE;
820
821 if (access_flags & IB_ACCESS_REMOTE_READ)
822 hw_access_flags |= MLX4_QP_BIT_RRE;
823 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
824 hw_access_flags |= MLX4_QP_BIT_RAE;
825 if (access_flags & IB_ACCESS_REMOTE_WRITE)
826 hw_access_flags |= MLX4_QP_BIT_RWE;
827
828 return cpu_to_be32(hw_access_flags);
829}
830
831static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
832 int attr_mask)
833{
834 if (attr_mask & IB_QP_PKEY_INDEX)
835 sqp->pkey_index = attr->pkey_index;
836 if (attr_mask & IB_QP_QKEY)
837 sqp->qkey = attr->qkey;
838 if (attr_mask & IB_QP_SQ_PSN)
839 sqp->send_psn = attr->sq_psn;
840}
841
842static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
843{
844 path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
845}
846
847static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
848 struct mlx4_qp_path *path, u8 port)
849{
850 path->grh_mylmc = ah->src_path_bits & 0x7f;
851 path->rlid = cpu_to_be16(ah->dlid);
852 if (ah->static_rate) {
853 path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET;
854 while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
855 !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
856 --path->static_rate;
857 } else
858 path->static_rate = 0;
859 path->counter_index = 0xff;
860
861 if (ah->ah_flags & IB_AH_GRH) {
862 if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
863 printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",
864 ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
865 return -1;
866 }
867
868 path->grh_mylmc |= 1 << 7;
869 path->mgid_index = ah->grh.sgid_index;
870 path->hop_limit = ah->grh.hop_limit;
871 path->tclass_flowlabel =
872 cpu_to_be32((ah->grh.traffic_class << 20) |
873 (ah->grh.flow_label));
874 memcpy(path->rgid, ah->grh.dgid.raw, 16);
875 }
876
877 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
878 ((port - 1) << 6) | ((ah->sl & 0xf) << 2);
879
880 return 0;
881}
882
883static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
884 const struct ib_qp_attr *attr, int attr_mask,
885 enum ib_qp_state cur_state, enum ib_qp_state new_state)
886{
887 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
888 struct mlx4_ib_qp *qp = to_mqp(ibqp);
889 struct mlx4_qp_context *context;
890 enum mlx4_qp_optpar optpar = 0;
891 int sqd_event;
892 int err = -EINVAL;
893
894 context = kzalloc(sizeof *context, GFP_KERNEL);
895 if (!context)
896 return -ENOMEM;
897
898 context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
899 (to_mlx4_st(ibqp->qp_type) << 16));
900 context->flags |= cpu_to_be32(1 << 8);
901
902 if (!(attr_mask & IB_QP_PATH_MIG_STATE))
903 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
904 else {
905 optpar |= MLX4_QP_OPTPAR_PM_STATE;
906 switch (attr->path_mig_state) {
907 case IB_MIG_MIGRATED:
908 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
909 break;
910 case IB_MIG_REARM:
911 context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
912 break;
913 case IB_MIG_ARMED:
914 context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
915 break;
916 }
917 }
918
919 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
920 context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
921 else if (ibqp->qp_type == IB_QPT_UD) {
922 if (qp->flags & MLX4_IB_QP_LSO)
923 context->mtu_msgmax = (IB_MTU_4096 << 5) |
924 ilog2(dev->dev->caps.max_gso_sz);
925 else
926 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
927 } else if (attr_mask & IB_QP_PATH_MTU) {
928 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
929 printk(KERN_ERR "path MTU (%u) is invalid\n",
930 attr->path_mtu);
931 goto out;
932 }
933 context->mtu_msgmax = (attr->path_mtu << 5) |
934 ilog2(dev->dev->caps.max_msg_sz);
935 }
936
937 if (qp->rq.wqe_cnt)
938 context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
939 context->rq_size_stride |= qp->rq.wqe_shift - 4;
940
941 if (qp->sq.wqe_cnt)
942 context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
943 context->sq_size_stride |= qp->sq.wqe_shift - 4;
944
945 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
946 context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
947
948 if (qp->ibqp.uobject)
949 context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index);
950 else
951 context->usr_page = cpu_to_be32(dev->priv_uar.index);
952
953 if (attr_mask & IB_QP_DEST_QPN)
954 context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
955
956 if (attr_mask & IB_QP_PORT) {
957 if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
958 !(attr_mask & IB_QP_AV)) {
959 mlx4_set_sched(&context->pri_path, attr->port_num);
960 optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
961 }
962 }
963
964 if (attr_mask & IB_QP_PKEY_INDEX) {
965 context->pri_path.pkey_index = attr->pkey_index;
966 optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
967 }
968
969 if (attr_mask & IB_QP_AV) {
970 if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,
971 attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
972 goto out;
973
974 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
975 MLX4_QP_OPTPAR_SCHED_QUEUE);
976 }
977
978 if (attr_mask & IB_QP_TIMEOUT) {
979 context->pri_path.ackto = attr->timeout << 3;
980 optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
981 }
982
983 if (attr_mask & IB_QP_ALT_PATH) {
984 if (attr->alt_port_num == 0 ||
985 attr->alt_port_num > dev->dev->caps.num_ports)
986 goto out;
987
988 if (attr->alt_pkey_index >=
989 dev->dev->caps.pkey_table_len[attr->alt_port_num])
990 goto out;
991
992 if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
993 attr->alt_port_num))
994 goto out;
995
996 context->alt_path.pkey_index = attr->alt_pkey_index;
997 context->alt_path.ackto = attr->alt_timeout << 3;
998 optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
999 }
1000
1001 context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn);
1002 context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
1003
1004
1005 if (!qp->ibqp.uobject)
1006 context->params1 |= cpu_to_be32(1 << 11);
1007
1008 if (attr_mask & IB_QP_RNR_RETRY) {
1009 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
1010 optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
1011 }
1012
1013 if (attr_mask & IB_QP_RETRY_CNT) {
1014 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
1015 optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
1016 }
1017
1018 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1019 if (attr->max_rd_atomic)
1020 context->params1 |=
1021 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
1022 optpar |= MLX4_QP_OPTPAR_SRA_MAX;
1023 }
1024
1025 if (attr_mask & IB_QP_SQ_PSN)
1026 context->next_send_psn = cpu_to_be32(attr->sq_psn);
1027
1028 context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn);
1029
1030 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1031 if (attr->max_dest_rd_atomic)
1032 context->params2 |=
1033 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
1034 optpar |= MLX4_QP_OPTPAR_RRA_MAX;
1035 }
1036
1037 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
1038 context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
1039 optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
1040 }
1041
1042 if (ibqp->srq)
1043 context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
1044
1045 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1046 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
1047 optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
1048 }
1049 if (attr_mask & IB_QP_RQ_PSN)
1050 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
1051
1052 context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn);
1053
1054 if (attr_mask & IB_QP_QKEY) {
1055 context->qkey = cpu_to_be32(attr->qkey);
1056 optpar |= MLX4_QP_OPTPAR_Q_KEY;
1057 }
1058
1059 if (ibqp->srq)
1060 context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
1061
1062 if (!ibqp->srq && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1063 context->db_rec_addr = cpu_to_be64(qp->db.dma);
1064
1065 if (cur_state == IB_QPS_INIT &&
1066 new_state == IB_QPS_RTR &&
1067 (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
1068 ibqp->qp_type == IB_QPT_UD)) {
1069 context->pri_path.sched_queue = (qp->port - 1) << 6;
1070 if (is_qp0(dev, qp))
1071 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
1072 else
1073 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
1074 }
1075
1076 if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD &&
1077 attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
1078 sqd_event = 1;
1079 else
1080 sqd_event = 0;
1081
1082 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1083 context->rlkey |= (1 << 4);
1084
1085
1086
1087
1088
1089
1090
1091 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1092 struct mlx4_wqe_ctrl_seg *ctrl;
1093 int i;
1094
1095 for (i = 0; i < qp->sq.wqe_cnt; ++i) {
1096 ctrl = get_send_wqe(qp, i);
1097 ctrl->owner_opcode = cpu_to_be32(1 << 31);
1098 if (qp->sq_max_wqes_per_wr == 1)
1099 ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4);
1100
1101 stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
1102 }
1103 }
1104
1105 err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
1106 to_mlx4_state(new_state), context, optpar,
1107 sqd_event, &qp->mqp);
1108 if (err)
1109 goto out;
1110
1111 qp->state = new_state;
1112
1113 if (attr_mask & IB_QP_ACCESS_FLAGS)
1114 qp->atomic_rd_en = attr->qp_access_flags;
1115 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1116 qp->resp_depth = attr->max_dest_rd_atomic;
1117 if (attr_mask & IB_QP_PORT)
1118 qp->port = attr->port_num;
1119 if (attr_mask & IB_QP_ALT_PATH)
1120 qp->alt_port = attr->alt_port_num;
1121
1122 if (is_sqp(dev, qp))
1123 store_sqp_attrs(to_msqp(qp), attr, attr_mask);
1124
1125
1126
1127
1128
1129 if (is_qp0(dev, qp)) {
1130 if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
1131 if (mlx4_INIT_PORT(dev->dev, qp->port))
1132 printk(KERN_WARNING "INIT_PORT failed for port %d\n",
1133 qp->port);
1134
1135 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
1136 (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
1137 mlx4_CLOSE_PORT(dev->dev, qp->port);
1138 }
1139
1140
1141
1142
1143
1144 if (new_state == IB_QPS_RESET && !ibqp->uobject) {
1145 mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn,
1146 ibqp->srq ? to_msrq(ibqp->srq): NULL);
1147 if (ibqp->send_cq != ibqp->recv_cq)
1148 mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL);
1149
1150 qp->rq.head = 0;
1151 qp->rq.tail = 0;
1152 qp->sq.head = 0;
1153 qp->sq.tail = 0;
1154 qp->sq_next_wqe = 0;
1155 if (!ibqp->srq)
1156 *qp->db.db = 0;
1157 }
1158
1159out:
1160 kfree(context);
1161 return err;
1162}
1163
1164int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1165 int attr_mask, struct ib_udata *udata)
1166{
1167 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
1168 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1169 enum ib_qp_state cur_state, new_state;
1170 int err = -EINVAL;
1171
1172 mutex_lock(&qp->mutex);
1173
1174 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
1175 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1176
1177 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))
1178 goto out;
1179
1180 if ((attr_mask & IB_QP_PORT) &&
1181 (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) {
1182 goto out;
1183 }
1184
1185 if (attr_mask & IB_QP_PKEY_INDEX) {
1186 int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1187 if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
1188 goto out;
1189 }
1190
1191 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1192 attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
1193 goto out;
1194 }
1195
1196 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1197 attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
1198 goto out;
1199 }
1200
1201 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1202 err = 0;
1203 goto out;
1204 }
1205
1206 err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
1207
1208out:
1209 mutex_unlock(&qp->mutex);
1210 return err;
1211}
1212
1213static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
1214 void *wqe, unsigned *mlx_seg_len)
1215{
1216 struct ib_device *ib_dev = &to_mdev(sqp->qp.ibqp.device)->ib_dev;
1217 struct mlx4_wqe_mlx_seg *mlx = wqe;
1218 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
1219 struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah);
1220 u16 pkey;
1221 int send_size;
1222 int header_size;
1223 int spc;
1224 int i;
1225
1226 send_size = 0;
1227 for (i = 0; i < wr->num_sge; ++i)
1228 send_size += wr->sg_list[i].length;
1229
1230 ib_ud_header_init(send_size, mlx4_ib_ah_grh_present(ah), &sqp->ud_header);
1231
1232 sqp->ud_header.lrh.service_level =
1233 be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 28;
1234 sqp->ud_header.lrh.destination_lid = ah->av.dlid;
1235 sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.g_slid & 0x7f);
1236 if (mlx4_ib_ah_grh_present(ah)) {
1237 sqp->ud_header.grh.traffic_class =
1238 (be32_to_cpu(ah->av.sl_tclass_flowlabel) >> 20) & 0xff;
1239 sqp->ud_header.grh.flow_label =
1240 ah->av.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
1241 sqp->ud_header.grh.hop_limit = ah->av.hop_limit;
1242 ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.port_pd) >> 24,
1243 ah->av.gid_index, &sqp->ud_header.grh.source_gid);
1244 memcpy(sqp->ud_header.grh.destination_gid.raw,
1245 ah->av.dgid, 16);
1246 }
1247
1248 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
1249 mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
1250 (sqp->ud_header.lrh.destination_lid ==
1251 IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
1252 (sqp->ud_header.lrh.service_level << 8));
1253 mlx->rlid = sqp->ud_header.lrh.destination_lid;
1254
1255 switch (wr->opcode) {
1256 case IB_WR_SEND:
1257 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
1258 sqp->ud_header.immediate_present = 0;
1259 break;
1260 case IB_WR_SEND_WITH_IMM:
1261 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1262 sqp->ud_header.immediate_present = 1;
1263 sqp->ud_header.immediate_data = wr->ex.imm_data;
1264 break;
1265 default:
1266 return -EINVAL;
1267 }
1268
1269 sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0;
1270 if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
1271 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
1272 sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED);
1273 if (!sqp->qp.ibqp.qp_num)
1274 ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
1275 else
1276 ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey);
1277 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
1278 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
1279 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
1280 sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ?
1281 sqp->qkey : wr->wr.ud.remote_qkey);
1282 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
1283
1284 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
1285
1286 if (0) {
1287 printk(KERN_ERR "built UD header of size %d:\n", header_size);
1288 for (i = 0; i < header_size / 4; ++i) {
1289 if (i % 8 == 0)
1290 printk(" [%02x] ", i * 4);
1291 printk(" %08x",
1292 be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
1293 if ((i + 1) % 8 == 0)
1294 printk("\n");
1295 }
1296 printk("\n");
1297 }
1298
1299
1300
1301
1302
1303
1304
1305 spc = MLX4_INLINE_ALIGN -
1306 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
1307 if (header_size <= spc) {
1308 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
1309 memcpy(inl + 1, sqp->header_buf, header_size);
1310 i = 1;
1311 } else {
1312 inl->byte_count = cpu_to_be32(1 << 31 | spc);
1313 memcpy(inl + 1, sqp->header_buf, spc);
1314
1315 inl = (void *) (inl + 1) + spc;
1316 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330 wmb();
1331 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
1332 i = 2;
1333 }
1334
1335 *mlx_seg_len =
1336 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
1337 return 0;
1338}
1339
1340static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
1341{
1342 unsigned cur;
1343 struct mlx4_ib_cq *cq;
1344
1345 cur = wq->head - wq->tail;
1346 if (likely(cur + nreq < wq->max_post))
1347 return 0;
1348
1349 cq = to_mcq(ib_cq);
1350 spin_lock(&cq->lock);
1351 cur = wq->head - wq->tail;
1352 spin_unlock(&cq->lock);
1353
1354 return cur + nreq >= wq->max_post;
1355}
1356
1357static __be32 convert_access(int acc)
1358{
1359 return (acc & IB_ACCESS_REMOTE_ATOMIC ? cpu_to_be32(MLX4_WQE_FMR_PERM_ATOMIC) : 0) |
1360 (acc & IB_ACCESS_REMOTE_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_WRITE) : 0) |
1361 (acc & IB_ACCESS_REMOTE_READ ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_READ) : 0) |
1362 (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) |
1363 cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
1364}
1365
1366static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr)
1367{
1368 struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list);
1369 int i;
1370
1371 for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i)
1372 mfrpl->mapped_page_list[i] =
1373 cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] |
1374 MLX4_MTT_FLAG_PRESENT);
1375
1376 fseg->flags = convert_access(wr->wr.fast_reg.access_flags);
1377 fseg->mem_key = cpu_to_be32(wr->wr.fast_reg.rkey);
1378 fseg->buf_list = cpu_to_be64(mfrpl->map);
1379 fseg->start_addr = cpu_to_be64(wr->wr.fast_reg.iova_start);
1380 fseg->reg_len = cpu_to_be64(wr->wr.fast_reg.length);
1381 fseg->offset = 0;
1382 fseg->page_size = cpu_to_be32(wr->wr.fast_reg.page_shift);
1383 fseg->reserved[0] = 0;
1384 fseg->reserved[1] = 0;
1385}
1386
1387static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
1388{
1389 iseg->flags = 0;
1390 iseg->mem_key = cpu_to_be32(rkey);
1391 iseg->guest_id = 0;
1392 iseg->pa = 0;
1393}
1394
1395static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
1396 u64 remote_addr, u32 rkey)
1397{
1398 rseg->raddr = cpu_to_be64(remote_addr);
1399 rseg->rkey = cpu_to_be32(rkey);
1400 rseg->reserved = 0;
1401}
1402
1403static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, struct ib_send_wr *wr)
1404{
1405 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
1406 aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap);
1407 aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add);
1408 } else {
1409 aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
1410 aseg->compare = 0;
1411 }
1412
1413}
1414
1415static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
1416 struct ib_send_wr *wr)
1417{
1418 memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
1419 dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn);
1420 dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey);
1421}
1422
1423static void set_mlx_icrc_seg(void *dseg)
1424{
1425 u32 *t = dseg;
1426 struct mlx4_wqe_inline_seg *iseg = dseg;
1427
1428 t[1] = 0;
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438 wmb();
1439
1440 iseg->byte_count = cpu_to_be32((1 << 31) | 4);
1441}
1442
1443static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
1444{
1445 dseg->lkey = cpu_to_be32(sg->lkey);
1446 dseg->addr = cpu_to_be64(sg->addr);
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456 wmb();
1457
1458 dseg->byte_count = cpu_to_be32(sg->length);
1459}
1460
1461static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
1462{
1463 dseg->byte_count = cpu_to_be32(sg->length);
1464 dseg->lkey = cpu_to_be32(sg->lkey);
1465 dseg->addr = cpu_to_be64(sg->addr);
1466}
1467
1468static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr,
1469 struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
1470 __be32 *lso_hdr_sz)
1471{
1472 unsigned halign = ALIGN(sizeof *wqe + wr->wr.ud.hlen, 16);
1473
1474
1475
1476
1477
1478 if (unlikely(halign > 64))
1479 return -EINVAL;
1480
1481 if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
1482 wr->num_sge > qp->sq.max_gs - (halign >> 4)))
1483 return -EINVAL;
1484
1485 memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen);
1486
1487 *lso_hdr_sz = cpu_to_be32((wr->wr.ud.mss - wr->wr.ud.hlen) << 16 |
1488 wr->wr.ud.hlen);
1489 *lso_seg_len = halign;
1490 return 0;
1491}
1492
1493static __be32 send_ieth(struct ib_send_wr *wr)
1494{
1495 switch (wr->opcode) {
1496 case IB_WR_SEND_WITH_IMM:
1497 case IB_WR_RDMA_WRITE_WITH_IMM:
1498 return wr->ex.imm_data;
1499
1500 case IB_WR_SEND_WITH_INV:
1501 return cpu_to_be32(wr->ex.invalidate_rkey);
1502
1503 default:
1504 return 0;
1505 }
1506}
1507
1508int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1509 struct ib_send_wr **bad_wr)
1510{
1511 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1512 void *wqe;
1513 struct mlx4_wqe_ctrl_seg *ctrl;
1514 struct mlx4_wqe_data_seg *dseg;
1515 unsigned long flags;
1516 int nreq;
1517 int err = 0;
1518 unsigned ind;
1519 int uninitialized_var(stamp);
1520 int uninitialized_var(size);
1521 unsigned uninitialized_var(seglen);
1522 __be32 dummy;
1523 __be32 *lso_wqe;
1524 __be32 uninitialized_var(lso_hdr_sz);
1525 int i;
1526
1527 spin_lock_irqsave(&qp->sq.lock, flags);
1528
1529 ind = qp->sq_next_wqe;
1530
1531 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1532 lso_wqe = &dummy;
1533
1534 if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
1535 err = -ENOMEM;
1536 *bad_wr = wr;
1537 goto out;
1538 }
1539
1540 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
1541 err = -EINVAL;
1542 *bad_wr = wr;
1543 goto out;
1544 }
1545
1546 ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
1547 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
1548
1549 ctrl->srcrb_flags =
1550 (wr->send_flags & IB_SEND_SIGNALED ?
1551 cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
1552 (wr->send_flags & IB_SEND_SOLICITED ?
1553 cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
1554 ((wr->send_flags & IB_SEND_IP_CSUM) ?
1555 cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
1556 MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
1557 qp->sq_signal_bits;
1558
1559 ctrl->imm = send_ieth(wr);
1560
1561 wqe += sizeof *ctrl;
1562 size = sizeof *ctrl / 16;
1563
1564 switch (ibqp->qp_type) {
1565 case IB_QPT_RC:
1566 case IB_QPT_UC:
1567 switch (wr->opcode) {
1568 case IB_WR_ATOMIC_CMP_AND_SWP:
1569 case IB_WR_ATOMIC_FETCH_AND_ADD:
1570 set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
1571 wr->wr.atomic.rkey);
1572 wqe += sizeof (struct mlx4_wqe_raddr_seg);
1573
1574 set_atomic_seg(wqe, wr);
1575 wqe += sizeof (struct mlx4_wqe_atomic_seg);
1576
1577 size += (sizeof (struct mlx4_wqe_raddr_seg) +
1578 sizeof (struct mlx4_wqe_atomic_seg)) / 16;
1579
1580 break;
1581
1582 case IB_WR_RDMA_READ:
1583 case IB_WR_RDMA_WRITE:
1584 case IB_WR_RDMA_WRITE_WITH_IMM:
1585 set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
1586 wr->wr.rdma.rkey);
1587 wqe += sizeof (struct mlx4_wqe_raddr_seg);
1588 size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
1589 break;
1590
1591 case IB_WR_LOCAL_INV:
1592 ctrl->srcrb_flags |=
1593 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
1594 set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
1595 wqe += sizeof (struct mlx4_wqe_local_inval_seg);
1596 size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
1597 break;
1598
1599 case IB_WR_FAST_REG_MR:
1600 ctrl->srcrb_flags |=
1601 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
1602 set_fmr_seg(wqe, wr);
1603 wqe += sizeof (struct mlx4_wqe_fmr_seg);
1604 size += sizeof (struct mlx4_wqe_fmr_seg) / 16;
1605 break;
1606
1607 default:
1608
1609 break;
1610 }
1611 break;
1612
1613 case IB_QPT_UD:
1614 set_datagram_seg(wqe, wr);
1615 wqe += sizeof (struct mlx4_wqe_datagram_seg);
1616 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
1617
1618 if (wr->opcode == IB_WR_LSO) {
1619 err = build_lso_seg(wqe, wr, qp, &seglen, &lso_hdr_sz);
1620 if (unlikely(err)) {
1621 *bad_wr = wr;
1622 goto out;
1623 }
1624 lso_wqe = (__be32 *) wqe;
1625 wqe += seglen;
1626 size += seglen / 16;
1627 }
1628 break;
1629
1630 case IB_QPT_SMI:
1631 case IB_QPT_GSI:
1632 err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen);
1633 if (unlikely(err)) {
1634 *bad_wr = wr;
1635 goto out;
1636 }
1637 wqe += seglen;
1638 size += seglen / 16;
1639 break;
1640
1641 default:
1642 break;
1643 }
1644
1645
1646
1647
1648
1649
1650
1651
1652 dseg = wqe;
1653 dseg += wr->num_sge - 1;
1654 size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
1655
1656
1657 if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI ||
1658 qp->ibqp.qp_type == IB_QPT_GSI)) {
1659 set_mlx_icrc_seg(dseg + 1);
1660 size += sizeof (struct mlx4_wqe_data_seg) / 16;
1661 }
1662
1663 for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
1664 set_data_seg(dseg, wr->sg_list + i);
1665
1666
1667
1668
1669
1670
1671 wmb();
1672 *lso_wqe = lso_hdr_sz;
1673
1674 ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ?
1675 MLX4_WQE_CTRL_FENCE : 0) | size;
1676
1677
1678
1679
1680
1681
1682 wmb();
1683
1684 if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
1685 err = -EINVAL;
1686 goto out;
1687 }
1688
1689 ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
1690 (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
1691
1692 stamp = ind + qp->sq_spare_wqes;
1693 ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704 if (wr->next) {
1705 stamp_send_wqe(qp, stamp, size * 16);
1706 ind = pad_wraparound(qp, ind);
1707 }
1708 }
1709
1710out:
1711 if (likely(nreq)) {
1712 qp->sq.head += nreq;
1713
1714
1715
1716
1717
1718 wmb();
1719
1720 writel(qp->doorbell_qpn,
1721 to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
1722
1723
1724
1725
1726
1727 mmiowb();
1728
1729 stamp_send_wqe(qp, stamp, size * 16);
1730
1731 ind = pad_wraparound(qp, ind);
1732 qp->sq_next_wqe = ind;
1733 }
1734
1735 spin_unlock_irqrestore(&qp->sq.lock, flags);
1736
1737 return err;
1738}
1739
1740int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
1741 struct ib_recv_wr **bad_wr)
1742{
1743 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1744 struct mlx4_wqe_data_seg *scat;
1745 unsigned long flags;
1746 int err = 0;
1747 int nreq;
1748 int ind;
1749 int i;
1750
1751 spin_lock_irqsave(&qp->rq.lock, flags);
1752
1753 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1754
1755 for (nreq = 0; wr; ++nreq, wr = wr->next) {
1756 if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.send_cq)) {
1757 err = -ENOMEM;
1758 *bad_wr = wr;
1759 goto out;
1760 }
1761
1762 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1763 err = -EINVAL;
1764 *bad_wr = wr;
1765 goto out;
1766 }
1767
1768 scat = get_recv_wqe(qp, ind);
1769
1770 for (i = 0; i < wr->num_sge; ++i)
1771 __set_data_seg(scat + i, wr->sg_list + i);
1772
1773 if (i < qp->rq.max_gs) {
1774 scat[i].byte_count = 0;
1775 scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
1776 scat[i].addr = 0;
1777 }
1778
1779 qp->rq.wrid[ind] = wr->wr_id;
1780
1781 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1782 }
1783
1784out:
1785 if (likely(nreq)) {
1786 qp->rq.head += nreq;
1787
1788
1789
1790
1791
1792 wmb();
1793
1794 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1795 }
1796
1797 spin_unlock_irqrestore(&qp->rq.lock, flags);
1798
1799 return err;
1800}
1801
1802static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
1803{
1804 switch (mlx4_state) {
1805 case MLX4_QP_STATE_RST: return IB_QPS_RESET;
1806 case MLX4_QP_STATE_INIT: return IB_QPS_INIT;
1807 case MLX4_QP_STATE_RTR: return IB_QPS_RTR;
1808 case MLX4_QP_STATE_RTS: return IB_QPS_RTS;
1809 case MLX4_QP_STATE_SQ_DRAINING:
1810 case MLX4_QP_STATE_SQD: return IB_QPS_SQD;
1811 case MLX4_QP_STATE_SQER: return IB_QPS_SQE;
1812 case MLX4_QP_STATE_ERR: return IB_QPS_ERR;
1813 default: return -1;
1814 }
1815}
1816
1817static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
1818{
1819 switch (mlx4_mig_state) {
1820 case MLX4_QP_PM_ARMED: return IB_MIG_ARMED;
1821 case MLX4_QP_PM_REARM: return IB_MIG_REARM;
1822 case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
1823 default: return -1;
1824 }
1825}
1826
1827static int to_ib_qp_access_flags(int mlx4_flags)
1828{
1829 int ib_flags = 0;
1830
1831 if (mlx4_flags & MLX4_QP_BIT_RRE)
1832 ib_flags |= IB_ACCESS_REMOTE_READ;
1833 if (mlx4_flags & MLX4_QP_BIT_RWE)
1834 ib_flags |= IB_ACCESS_REMOTE_WRITE;
1835 if (mlx4_flags & MLX4_QP_BIT_RAE)
1836 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
1837
1838 return ib_flags;
1839}
1840
1841static void to_ib_ah_attr(struct mlx4_dev *dev, struct ib_ah_attr *ib_ah_attr,
1842 struct mlx4_qp_path *path)
1843{
1844 memset(ib_ah_attr, 0, sizeof *ib_ah_attr);
1845 ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1;
1846
1847 if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports)
1848 return;
1849
1850 ib_ah_attr->dlid = be16_to_cpu(path->rlid);
1851 ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf;
1852 ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f;
1853 ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0;
1854 ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
1855 if (ib_ah_attr->ah_flags) {
1856 ib_ah_attr->grh.sgid_index = path->mgid_index;
1857 ib_ah_attr->grh.hop_limit = path->hop_limit;
1858 ib_ah_attr->grh.traffic_class =
1859 (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff;
1860 ib_ah_attr->grh.flow_label =
1861 be32_to_cpu(path->tclass_flowlabel) & 0xfffff;
1862 memcpy(ib_ah_attr->grh.dgid.raw,
1863 path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
1864 }
1865}
1866
1867int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
1868 struct ib_qp_init_attr *qp_init_attr)
1869{
1870 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
1871 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1872 struct mlx4_qp_context context;
1873 int mlx4_state;
1874 int err = 0;
1875
1876 mutex_lock(&qp->mutex);
1877
1878 if (qp->state == IB_QPS_RESET) {
1879 qp_attr->qp_state = IB_QPS_RESET;
1880 goto done;
1881 }
1882
1883 err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
1884 if (err) {
1885 err = -EINVAL;
1886 goto out;
1887 }
1888
1889 mlx4_state = be32_to_cpu(context.flags) >> 28;
1890
1891 qp->state = to_ib_qp_state(mlx4_state);
1892 qp_attr->qp_state = qp->state;
1893 qp_attr->path_mtu = context.mtu_msgmax >> 5;
1894 qp_attr->path_mig_state =
1895 to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
1896 qp_attr->qkey = be32_to_cpu(context.qkey);
1897 qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
1898 qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff;
1899 qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff;
1900 qp_attr->qp_access_flags =
1901 to_ib_qp_access_flags(be32_to_cpu(context.params2));
1902
1903 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
1904 to_ib_ah_attr(dev->dev, &qp_attr->ah_attr, &context.pri_path);
1905 to_ib_ah_attr(dev->dev, &qp_attr->alt_ah_attr, &context.alt_path);
1906 qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
1907 qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num;
1908 }
1909
1910 qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
1911 if (qp_attr->qp_state == IB_QPS_INIT)
1912 qp_attr->port_num = qp->port;
1913 else
1914 qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
1915
1916
1917 qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
1918
1919 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
1920
1921 qp_attr->max_dest_rd_atomic =
1922 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
1923 qp_attr->min_rnr_timer =
1924 (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
1925 qp_attr->timeout = context.pri_path.ackto >> 3;
1926 qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7;
1927 qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7;
1928 qp_attr->alt_timeout = context.alt_path.ackto >> 3;
1929
1930done:
1931 qp_attr->cur_qp_state = qp_attr->qp_state;
1932 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
1933 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
1934
1935 if (!ibqp->uobject) {
1936 qp_attr->cap.max_send_wr = qp->sq.wqe_cnt;
1937 qp_attr->cap.max_send_sge = qp->sq.max_gs;
1938 } else {
1939 qp_attr->cap.max_send_wr = 0;
1940 qp_attr->cap.max_send_sge = 0;
1941 }
1942
1943
1944
1945
1946
1947 qp_attr->cap.max_inline_data = 0;
1948
1949 qp_init_attr->cap = qp_attr->cap;
1950
1951 qp_init_attr->create_flags = 0;
1952 if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
1953 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
1954
1955 if (qp->flags & MLX4_IB_QP_LSO)
1956 qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
1957
1958out:
1959 mutex_unlock(&qp->mutex);
1960 return err;
1961}
1962
1963