1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/log2.h>
35#include <linux/etherdevice.h>
36#include <net/ip.h>
37#include <linux/slab.h>
38#include <linux/netdevice.h>
39#include <linux/vmalloc.h>
40
41#include <rdma/ib_cache.h>
42#include <rdma/ib_pack.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_mad.h>
45
46#include <linux/mlx4/driver.h>
47#include <linux/mlx4/qp.h>
48
49#include "mlx4_ib.h"
50#include "user.h"
51
52static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
53 struct mlx4_ib_cq *recv_cq);
54static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
55 struct mlx4_ib_cq *recv_cq);
56
57enum {
58 MLX4_IB_ACK_REQ_FREQ = 8,
59};
60
61enum {
62 MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,
63 MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
64 MLX4_IB_LINK_TYPE_IB = 0,
65 MLX4_IB_LINK_TYPE_ETH = 1
66};
67
68enum {
69
70
71
72
73
74
75 MLX4_IB_UD_HEADER_SIZE = 82,
76 MLX4_IB_LSO_HEADER_SPARE = 128,
77};
78
79enum {
80 MLX4_IB_IBOE_ETHERTYPE = 0x8915
81};
82
83struct mlx4_ib_sqp {
84 struct mlx4_ib_qp qp;
85 int pkey_index;
86 u32 qkey;
87 u32 send_psn;
88 struct ib_ud_header ud_header;
89 u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
90 struct ib_qp *roce_v2_gsi;
91};
92
93enum {
94 MLX4_IB_MIN_SQ_STRIDE = 6,
95 MLX4_IB_CACHE_LINE_SIZE = 64,
96};
97
98enum {
99 MLX4_RAW_QP_MTU = 7,
100 MLX4_RAW_QP_MSGMAX = 31,
101};
102
103#ifndef ETH_ALEN
104#define ETH_ALEN 6
105#endif
106
107static const __be32 mlx4_ib_opcode[] = {
108 [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND),
109 [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO),
110 [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM),
111 [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
112 [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
113 [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ),
114 [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
115 [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
116 [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
117 [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
118 [IB_WR_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR),
119 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
120 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
121};
122
123static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
124{
125 return container_of(mqp, struct mlx4_ib_sqp, qp);
126}
127
128static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
129{
130 if (!mlx4_is_master(dev->dev))
131 return 0;
132
133 return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn &&
134 qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn +
135 8 * MLX4_MFUNC_MAX;
136}
137
138static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
139{
140 int proxy_sqp = 0;
141 int real_sqp = 0;
142 int i;
143
144 real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
145 qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
146 qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3);
147 if (real_sqp)
148 return 1;
149
150 if (mlx4_is_mfunc(dev->dev)) {
151 for (i = 0; i < dev->dev->caps.num_ports; i++) {
152 if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i] ||
153 qp->mqp.qpn == dev->dev->caps.qp1_proxy[i]) {
154 proxy_sqp = 1;
155 break;
156 }
157 }
158 }
159 if (proxy_sqp)
160 return 1;
161
162 return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP);
163}
164
165
166static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
167{
168 int proxy_qp0 = 0;
169 int real_qp0 = 0;
170 int i;
171
172 real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
173 qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
174 qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1);
175 if (real_qp0)
176 return 1;
177
178 if (mlx4_is_mfunc(dev->dev)) {
179 for (i = 0; i < dev->dev->caps.num_ports; i++) {
180 if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i]) {
181 proxy_qp0 = 1;
182 break;
183 }
184 }
185 }
186 return proxy_qp0;
187}
188
189static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
190{
191 return mlx4_buf_offset(&qp->buf, offset);
192}
193
194static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
195{
196 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
197}
198
199static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
200{
201 return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
202}
203
204
205
206
207
208
209
210
211
212
213static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
214{
215 __be32 *wqe;
216 int i;
217 int s;
218 int ind;
219 void *buf;
220 __be32 stamp;
221 struct mlx4_wqe_ctrl_seg *ctrl;
222
223 if (qp->sq_max_wqes_per_wr > 1) {
224 s = roundup(size, 1U << qp->sq.wqe_shift);
225 for (i = 0; i < s; i += 64) {
226 ind = (i >> qp->sq.wqe_shift) + n;
227 stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
228 cpu_to_be32(0xffffffff);
229 buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
230 wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
231 *wqe = stamp;
232 }
233 } else {
234 ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
235 s = (ctrl->fence_size & 0x3f) << 4;
236 for (i = 64; i < s; i += 64) {
237 wqe = buf + i;
238 *wqe = cpu_to_be32(0xffffffff);
239 }
240 }
241}
242
243static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
244{
245 struct mlx4_wqe_ctrl_seg *ctrl;
246 struct mlx4_wqe_inline_seg *inl;
247 void *wqe;
248 int s;
249
250 ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
251 s = sizeof(struct mlx4_wqe_ctrl_seg);
252
253 if (qp->ibqp.qp_type == IB_QPT_UD) {
254 struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
255 struct mlx4_av *av = (struct mlx4_av *)dgram->av;
256 memset(dgram, 0, sizeof *dgram);
257 av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
258 s += sizeof(struct mlx4_wqe_datagram_seg);
259 }
260
261
262 if (size > s) {
263 inl = wqe + s;
264 inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
265 }
266 ctrl->srcrb_flags = 0;
267 ctrl->fence_size = size / 16;
268
269
270
271
272 wmb();
273
274 ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
275 (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
276
277 stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
278}
279
280
281static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
282{
283 unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
284 if (unlikely(s < qp->sq_max_wqes_per_wr)) {
285 post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
286 ind += s;
287 }
288 return ind;
289}
290
291static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
292{
293 struct ib_event event;
294 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
295
296 if (type == MLX4_EVENT_TYPE_PATH_MIG)
297 to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
298
299 if (ibqp->event_handler) {
300 event.device = ibqp->device;
301 event.element.qp = ibqp;
302 switch (type) {
303 case MLX4_EVENT_TYPE_PATH_MIG:
304 event.event = IB_EVENT_PATH_MIG;
305 break;
306 case MLX4_EVENT_TYPE_COMM_EST:
307 event.event = IB_EVENT_COMM_EST;
308 break;
309 case MLX4_EVENT_TYPE_SQ_DRAINED:
310 event.event = IB_EVENT_SQ_DRAINED;
311 break;
312 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
313 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
314 break;
315 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
316 event.event = IB_EVENT_QP_FATAL;
317 break;
318 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
319 event.event = IB_EVENT_PATH_MIG_ERR;
320 break;
321 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
322 event.event = IB_EVENT_QP_REQ_ERR;
323 break;
324 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
325 event.event = IB_EVENT_QP_ACCESS_ERR;
326 break;
327 default:
328 pr_warn("Unexpected event type %d "
329 "on QP %06x\n", type, qp->qpn);
330 return;
331 }
332
333 ibqp->event_handler(&event, ibqp->qp_context);
334 }
335}
336
337static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
338{
339
340
341
342
343
344
345 switch (type) {
346 case MLX4_IB_QPT_UD:
347 return sizeof (struct mlx4_wqe_ctrl_seg) +
348 sizeof (struct mlx4_wqe_datagram_seg) +
349 ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
350 case MLX4_IB_QPT_PROXY_SMI_OWNER:
351 case MLX4_IB_QPT_PROXY_SMI:
352 case MLX4_IB_QPT_PROXY_GSI:
353 return sizeof (struct mlx4_wqe_ctrl_seg) +
354 sizeof (struct mlx4_wqe_datagram_seg) + 64;
355 case MLX4_IB_QPT_TUN_SMI_OWNER:
356 case MLX4_IB_QPT_TUN_GSI:
357 return sizeof (struct mlx4_wqe_ctrl_seg) +
358 sizeof (struct mlx4_wqe_datagram_seg);
359
360 case MLX4_IB_QPT_UC:
361 return sizeof (struct mlx4_wqe_ctrl_seg) +
362 sizeof (struct mlx4_wqe_raddr_seg);
363 case MLX4_IB_QPT_RC:
364 return sizeof (struct mlx4_wqe_ctrl_seg) +
365 sizeof (struct mlx4_wqe_atomic_seg) +
366 sizeof (struct mlx4_wqe_raddr_seg);
367 case MLX4_IB_QPT_SMI:
368 case MLX4_IB_QPT_GSI:
369 return sizeof (struct mlx4_wqe_ctrl_seg) +
370 ALIGN(MLX4_IB_UD_HEADER_SIZE +
371 DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
372 MLX4_INLINE_ALIGN) *
373 sizeof (struct mlx4_wqe_inline_seg),
374 sizeof (struct mlx4_wqe_data_seg)) +
375 ALIGN(4 +
376 sizeof (struct mlx4_wqe_inline_seg),
377 sizeof (struct mlx4_wqe_data_seg));
378 default:
379 return sizeof (struct mlx4_wqe_ctrl_seg);
380 }
381}
382
383static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
384 int is_user, int has_rq, struct mlx4_ib_qp *qp)
385{
386
387 if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
388 cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
389 return -EINVAL;
390
391 if (!has_rq) {
392 if (cap->max_recv_wr)
393 return -EINVAL;
394
395 qp->rq.wqe_cnt = qp->rq.max_gs = 0;
396 } else {
397
398 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
399 return -EINVAL;
400
401 qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));
402 qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));
403 qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
404 }
405
406
407 if (is_user) {
408 cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
409 cap->max_recv_sge = qp->rq.max_gs;
410 } else {
411 cap->max_recv_wr = qp->rq.max_post =
412 min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
413 cap->max_recv_sge = min(qp->rq.max_gs,
414 min(dev->dev->caps.max_sq_sg,
415 dev->dev->caps.max_rq_sg));
416 }
417
418 return 0;
419}
420
421static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
422 enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp)
423{
424 int s;
425
426
427 if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) ||
428 cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
429 cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
430 sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
431 return -EINVAL;
432
433
434
435
436
437 if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI ||
438 type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) &&
439 cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
440 return -EINVAL;
441
442 s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
443 cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
444 send_wqe_overhead(type, qp->flags);
445
446 if (s > dev->dev->caps.max_sq_desc_sz)
447 return -EINVAL;
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480 if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
481 qp->sq_signal_bits && BITS_PER_LONG == 64 &&
482 type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI &&
483 !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI |
484 MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER)))
485 qp->sq.wqe_shift = ilog2(64);
486 else
487 qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
488
489 for (;;) {
490 qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
491
492
493
494
495
496 qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
497 qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
498 qp->sq_max_wqes_per_wr +
499 qp->sq_spare_wqes);
500
501 if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
502 break;
503
504 if (qp->sq_max_wqes_per_wr <= 1)
505 return -EINVAL;
506
507 ++qp->sq.wqe_shift;
508 }
509
510 qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
511 (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
512 send_wqe_overhead(type, qp->flags)) /
513 sizeof (struct mlx4_wqe_data_seg);
514
515 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
516 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
517 if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
518 qp->rq.offset = 0;
519 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
520 } else {
521 qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
522 qp->sq.offset = 0;
523 }
524
525 cap->max_send_wr = qp->sq.max_post =
526 (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
527 cap->max_send_sge = min(qp->sq.max_gs,
528 min(dev->dev->caps.max_sq_sg,
529 dev->dev->caps.max_rq_sg));
530
531 cap->max_inline_data = 0;
532
533 return 0;
534}
535
536static int set_user_sq_size(struct mlx4_ib_dev *dev,
537 struct mlx4_ib_qp *qp,
538 struct mlx4_ib_create_qp *ucmd)
539{
540
541 if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||
542 ucmd->log_sq_stride >
543 ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
544 ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
545 return -EINVAL;
546
547 qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
548 qp->sq.wqe_shift = ucmd->log_sq_stride;
549
550 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
551 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
552
553 return 0;
554}
555
556static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
557{
558 int i;
559
560 qp->sqp_proxy_rcv =
561 kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt,
562 GFP_KERNEL);
563 if (!qp->sqp_proxy_rcv)
564 return -ENOMEM;
565 for (i = 0; i < qp->rq.wqe_cnt; i++) {
566 qp->sqp_proxy_rcv[i].addr =
567 kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr),
568 GFP_KERNEL);
569 if (!qp->sqp_proxy_rcv[i].addr)
570 goto err;
571 qp->sqp_proxy_rcv[i].map =
572 ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr,
573 sizeof (struct mlx4_ib_proxy_sqp_hdr),
574 DMA_FROM_DEVICE);
575 if (ib_dma_mapping_error(dev, qp->sqp_proxy_rcv[i].map)) {
576 kfree(qp->sqp_proxy_rcv[i].addr);
577 goto err;
578 }
579 }
580 return 0;
581
582err:
583 while (i > 0) {
584 --i;
585 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
586 sizeof (struct mlx4_ib_proxy_sqp_hdr),
587 DMA_FROM_DEVICE);
588 kfree(qp->sqp_proxy_rcv[i].addr);
589 }
590 kfree(qp->sqp_proxy_rcv);
591 qp->sqp_proxy_rcv = NULL;
592 return -ENOMEM;
593}
594
595static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
596{
597 int i;
598
599 for (i = 0; i < qp->rq.wqe_cnt; i++) {
600 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
601 sizeof (struct mlx4_ib_proxy_sqp_hdr),
602 DMA_FROM_DEVICE);
603 kfree(qp->sqp_proxy_rcv[i].addr);
604 }
605 kfree(qp->sqp_proxy_rcv);
606}
607
608static int qp_has_rq(struct ib_qp_init_attr *attr)
609{
610 if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT)
611 return 0;
612
613 return !attr->srq;
614}
615
616static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
617{
618 int i;
619 for (i = 0; i < dev->caps.num_ports; i++) {
620 if (qpn == dev->caps.qp0_proxy[i])
621 return !!dev->caps.qp0_qkey[i];
622 }
623 return 0;
624}
625
626static void mlx4_ib_free_qp_counter(struct mlx4_ib_dev *dev,
627 struct mlx4_ib_qp *qp)
628{
629 mutex_lock(&dev->counters_table[qp->port - 1].mutex);
630 mlx4_counter_free(dev->dev, qp->counter_index->index);
631 list_del(&qp->counter_index->list);
632 mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
633
634 kfree(qp->counter_index);
635 qp->counter_index = NULL;
636}
637
638static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
639 struct ib_qp_init_attr *init_attr,
640 struct ib_udata *udata, int sqpn, struct mlx4_ib_qp **caller_qp,
641 gfp_t gfp)
642{
643 int qpn;
644 int err;
645 struct mlx4_ib_sqp *sqp;
646 struct mlx4_ib_qp *qp;
647 enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
648 struct mlx4_ib_cq *mcq;
649 unsigned long flags;
650
651
652 if (sqpn) {
653 if (mlx4_is_mfunc(dev->dev) &&
654 (!mlx4_is_master(dev->dev) ||
655 !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {
656 if (init_attr->qp_type == IB_QPT_GSI)
657 qp_type = MLX4_IB_QPT_PROXY_GSI;
658 else {
659 if (mlx4_is_master(dev->dev) ||
660 qp0_enabled_vf(dev->dev, sqpn))
661 qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;
662 else
663 qp_type = MLX4_IB_QPT_PROXY_SMI;
664 }
665 }
666 qpn = sqpn;
667
668 init_attr->cap.max_recv_sge++;
669 } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) {
670 struct mlx4_ib_qp_tunnel_init_attr *tnl_init =
671 container_of(init_attr,
672 struct mlx4_ib_qp_tunnel_init_attr, init_attr);
673 if ((tnl_init->proxy_qp_type != IB_QPT_SMI &&
674 tnl_init->proxy_qp_type != IB_QPT_GSI) ||
675 !mlx4_is_master(dev->dev))
676 return -EINVAL;
677 if (tnl_init->proxy_qp_type == IB_QPT_GSI)
678 qp_type = MLX4_IB_QPT_TUN_GSI;
679 else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||
680 mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,
681 tnl_init->port))
682 qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;
683 else
684 qp_type = MLX4_IB_QPT_TUN_SMI;
685
686
687 qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave
688 + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1;
689 sqpn = qpn;
690 }
691
692 if (!*caller_qp) {
693 if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI ||
694 (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |
695 MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {
696 sqp = kzalloc(sizeof (struct mlx4_ib_sqp), gfp);
697 if (!sqp)
698 return -ENOMEM;
699 qp = &sqp->qp;
700 qp->pri.vid = 0xFFFF;
701 qp->alt.vid = 0xFFFF;
702 } else {
703 qp = kzalloc(sizeof (struct mlx4_ib_qp), gfp);
704 if (!qp)
705 return -ENOMEM;
706 qp->pri.vid = 0xFFFF;
707 qp->alt.vid = 0xFFFF;
708 }
709 } else
710 qp = *caller_qp;
711
712 qp->mlx4_ib_qp_type = qp_type;
713
714 mutex_init(&qp->mutex);
715 spin_lock_init(&qp->sq.lock);
716 spin_lock_init(&qp->rq.lock);
717 INIT_LIST_HEAD(&qp->gid_list);
718 INIT_LIST_HEAD(&qp->steering_rules);
719
720 qp->state = IB_QPS_RESET;
721 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
722 qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
723
724 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, qp_has_rq(init_attr), qp);
725 if (err)
726 goto err;
727
728 if (pd->uobject) {
729 struct mlx4_ib_create_qp ucmd;
730
731 if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
732 err = -EFAULT;
733 goto err;
734 }
735
736 qp->sq_no_prefetch = ucmd.sq_no_prefetch;
737
738 err = set_user_sq_size(dev, qp, &ucmd);
739 if (err)
740 goto err;
741
742 qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
743 qp->buf_size, 0, 0);
744 if (IS_ERR(qp->umem)) {
745 err = PTR_ERR(qp->umem);
746 goto err;
747 }
748
749 err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem),
750 ilog2(qp->umem->page_size), &qp->mtt);
751 if (err)
752 goto err_buf;
753
754 err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
755 if (err)
756 goto err_mtt;
757
758 if (qp_has_rq(init_attr)) {
759 err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
760 ucmd.db_addr, &qp->db);
761 if (err)
762 goto err_mtt;
763 }
764 } else {
765 qp->sq_no_prefetch = 0;
766
767 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
768 qp->flags |= MLX4_IB_QP_LSO;
769
770 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
771 if (dev->steering_support ==
772 MLX4_STEERING_MODE_DEVICE_MANAGED)
773 qp->flags |= MLX4_IB_QP_NETIF;
774 else
775 goto err;
776 }
777
778 err = set_kernel_sq_size(dev, &init_attr->cap, qp_type, qp);
779 if (err)
780 goto err;
781
782 if (qp_has_rq(init_attr)) {
783 err = mlx4_db_alloc(dev->dev, &qp->db, 0, gfp);
784 if (err)
785 goto err;
786
787 *qp->db.db = 0;
788 }
789
790 if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf, gfp)) {
791 err = -ENOMEM;
792 goto err_db;
793 }
794
795 err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
796 &qp->mtt);
797 if (err)
798 goto err_buf;
799
800 err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf, gfp);
801 if (err)
802 goto err_mtt;
803
804 qp->sq.wrid = kmalloc_array(qp->sq.wqe_cnt, sizeof(u64),
805 gfp | __GFP_NOWARN);
806 if (!qp->sq.wrid)
807 qp->sq.wrid = __vmalloc(qp->sq.wqe_cnt * sizeof(u64),
808 gfp, PAGE_KERNEL);
809 qp->rq.wrid = kmalloc_array(qp->rq.wqe_cnt, sizeof(u64),
810 gfp | __GFP_NOWARN);
811 if (!qp->rq.wrid)
812 qp->rq.wrid = __vmalloc(qp->rq.wqe_cnt * sizeof(u64),
813 gfp, PAGE_KERNEL);
814 if (!qp->sq.wrid || !qp->rq.wrid) {
815 err = -ENOMEM;
816 goto err_wrid;
817 }
818 }
819
820 if (sqpn) {
821 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
822 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
823 if (alloc_proxy_bufs(pd->device, qp)) {
824 err = -ENOMEM;
825 goto err_wrid;
826 }
827 }
828 } else {
829
830
831
832 if (init_attr->qp_type == IB_QPT_RAW_PACKET)
833 err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
834 (init_attr->cap.max_send_wr ?
835 MLX4_RESERVE_ETH_BF_QP : 0) |
836 (init_attr->cap.max_recv_wr ?
837 MLX4_RESERVE_A0_QP : 0));
838 else
839 if (qp->flags & MLX4_IB_QP_NETIF)
840 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
841 else
842 err = mlx4_qp_reserve_range(dev->dev, 1, 1,
843 &qpn, 0);
844 if (err)
845 goto err_proxy;
846 }
847
848 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
849 qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
850
851 err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp, gfp);
852 if (err)
853 goto err_qpn;
854
855 if (init_attr->qp_type == IB_QPT_XRC_TGT)
856 qp->mqp.qpn |= (1 << 23);
857
858
859
860
861
862
863 qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
864
865 qp->mqp.event = mlx4_ib_qp_event;
866 if (!*caller_qp)
867 *caller_qp = qp;
868
869 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
870 mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
871 to_mcq(init_attr->recv_cq));
872
873
874
875 list_add_tail(&qp->qps_list, &dev->qp_list);
876
877
878
879 mcq = to_mcq(init_attr->send_cq);
880 list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
881 mcq = to_mcq(init_attr->recv_cq);
882 list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
883 mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
884 to_mcq(init_attr->recv_cq));
885 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
886 return 0;
887
888err_qpn:
889 if (!sqpn) {
890 if (qp->flags & MLX4_IB_QP_NETIF)
891 mlx4_ib_steer_qp_free(dev, qpn, 1);
892 else
893 mlx4_qp_release_range(dev->dev, qpn, 1);
894 }
895err_proxy:
896 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
897 free_proxy_bufs(pd->device, qp);
898err_wrid:
899 if (pd->uobject) {
900 if (qp_has_rq(init_attr))
901 mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
902 } else {
903 kvfree(qp->sq.wrid);
904 kvfree(qp->rq.wrid);
905 }
906
907err_mtt:
908 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
909
910err_buf:
911 if (pd->uobject)
912 ib_umem_release(qp->umem);
913 else
914 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
915
916err_db:
917 if (!pd->uobject && qp_has_rq(init_attr))
918 mlx4_db_free(dev->dev, &qp->db);
919
920err:
921 if (!*caller_qp)
922 kfree(qp);
923 return err;
924}
925
926static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
927{
928 switch (state) {
929 case IB_QPS_RESET: return MLX4_QP_STATE_RST;
930 case IB_QPS_INIT: return MLX4_QP_STATE_INIT;
931 case IB_QPS_RTR: return MLX4_QP_STATE_RTR;
932 case IB_QPS_RTS: return MLX4_QP_STATE_RTS;
933 case IB_QPS_SQD: return MLX4_QP_STATE_SQD;
934 case IB_QPS_SQE: return MLX4_QP_STATE_SQER;
935 case IB_QPS_ERR: return MLX4_QP_STATE_ERR;
936 default: return -1;
937 }
938}
939
940static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
941 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
942{
943 if (send_cq == recv_cq) {
944 spin_lock(&send_cq->lock);
945 __acquire(&recv_cq->lock);
946 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
947 spin_lock(&send_cq->lock);
948 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
949 } else {
950 spin_lock(&recv_cq->lock);
951 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
952 }
953}
954
955static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
956 __releases(&send_cq->lock) __releases(&recv_cq->lock)
957{
958 if (send_cq == recv_cq) {
959 __release(&recv_cq->lock);
960 spin_unlock(&send_cq->lock);
961 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
962 spin_unlock(&recv_cq->lock);
963 spin_unlock(&send_cq->lock);
964 } else {
965 spin_unlock(&send_cq->lock);
966 spin_unlock(&recv_cq->lock);
967 }
968}
969
970static void del_gid_entries(struct mlx4_ib_qp *qp)
971{
972 struct mlx4_ib_gid_entry *ge, *tmp;
973
974 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
975 list_del(&ge->list);
976 kfree(ge);
977 }
978}
979
980static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
981{
982 if (qp->ibqp.qp_type == IB_QPT_XRC_TGT)
983 return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd);
984 else
985 return to_mpd(qp->ibqp.pd);
986}
987
988static void get_cqs(struct mlx4_ib_qp *qp,
989 struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
990{
991 switch (qp->ibqp.qp_type) {
992 case IB_QPT_XRC_TGT:
993 *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq);
994 *recv_cq = *send_cq;
995 break;
996 case IB_QPT_XRC_INI:
997 *send_cq = to_mcq(qp->ibqp.send_cq);
998 *recv_cq = *send_cq;
999 break;
1000 default:
1001 *send_cq = to_mcq(qp->ibqp.send_cq);
1002 *recv_cq = to_mcq(qp->ibqp.recv_cq);
1003 break;
1004 }
1005}
1006
1007static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
1008 int is_user)
1009{
1010 struct mlx4_ib_cq *send_cq, *recv_cq;
1011 unsigned long flags;
1012
1013 if (qp->state != IB_QPS_RESET) {
1014 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1015 MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1016 pr_warn("modify QP %06x to RESET failed.\n",
1017 qp->mqp.qpn);
1018 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
1019 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
1020 qp->pri.smac = 0;
1021 qp->pri.smac_port = 0;
1022 }
1023 if (qp->alt.smac) {
1024 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
1025 qp->alt.smac = 0;
1026 }
1027 if (qp->pri.vid < 0x1000) {
1028 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
1029 qp->pri.vid = 0xFFFF;
1030 qp->pri.candidate_vid = 0xFFFF;
1031 qp->pri.update_vid = 0;
1032 }
1033 if (qp->alt.vid < 0x1000) {
1034 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
1035 qp->alt.vid = 0xFFFF;
1036 qp->alt.candidate_vid = 0xFFFF;
1037 qp->alt.update_vid = 0;
1038 }
1039 }
1040
1041 get_cqs(qp, &send_cq, &recv_cq);
1042
1043 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1044 mlx4_ib_lock_cqs(send_cq, recv_cq);
1045
1046
1047 list_del(&qp->qps_list);
1048 list_del(&qp->cq_send_list);
1049 list_del(&qp->cq_recv_list);
1050 if (!is_user) {
1051 __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1052 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1053 if (send_cq != recv_cq)
1054 __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
1055 }
1056
1057 mlx4_qp_remove(dev->dev, &qp->mqp);
1058
1059 mlx4_ib_unlock_cqs(send_cq, recv_cq);
1060 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1061
1062 mlx4_qp_free(dev->dev, &qp->mqp);
1063
1064 if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
1065 if (qp->flags & MLX4_IB_QP_NETIF)
1066 mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1067 else
1068 mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1069 }
1070
1071 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1072
1073 if (is_user) {
1074 if (qp->rq.wqe_cnt)
1075 mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
1076 &qp->db);
1077 ib_umem_release(qp->umem);
1078 } else {
1079 kvfree(qp->sq.wrid);
1080 kvfree(qp->rq.wrid);
1081 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1082 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
1083 free_proxy_bufs(&dev->ib_dev, qp);
1084 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1085 if (qp->rq.wqe_cnt)
1086 mlx4_db_free(dev->dev, &qp->db);
1087 }
1088
1089 del_gid_entries(qp);
1090}
1091
1092static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
1093{
1094
1095 if (!mlx4_is_mfunc(dev->dev) ||
1096 (mlx4_is_master(dev->dev) &&
1097 attr->create_flags & MLX4_IB_SRIOV_SQP)) {
1098 return dev->dev->phys_caps.base_sqpn +
1099 (attr->qp_type == IB_QPT_SMI ? 0 : 2) +
1100 attr->port_num - 1;
1101 }
1102
1103 if (attr->qp_type == IB_QPT_SMI)
1104 return dev->dev->caps.qp0_proxy[attr->port_num - 1];
1105 else
1106 return dev->dev->caps.qp1_proxy[attr->port_num - 1];
1107}
1108
1109static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
1110 struct ib_qp_init_attr *init_attr,
1111 struct ib_udata *udata)
1112{
1113 struct mlx4_ib_qp *qp = NULL;
1114 int err;
1115 int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1116 u16 xrcdn = 0;
1117 gfp_t gfp;
1118
1119 gfp = (init_attr->create_flags & MLX4_IB_QP_CREATE_USE_GFP_NOIO) ?
1120 GFP_NOIO : GFP_KERNEL;
1121
1122
1123
1124
1125 if (init_attr->create_flags & ~(MLX4_IB_QP_LSO |
1126 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK |
1127 MLX4_IB_SRIOV_TUNNEL_QP |
1128 MLX4_IB_SRIOV_SQP |
1129 MLX4_IB_QP_NETIF |
1130 MLX4_IB_QP_CREATE_ROCE_V2_GSI |
1131 MLX4_IB_QP_CREATE_USE_GFP_NOIO))
1132 return ERR_PTR(-EINVAL);
1133
1134 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1135 if (init_attr->qp_type != IB_QPT_UD)
1136 return ERR_PTR(-EINVAL);
1137 }
1138
1139 if (init_attr->create_flags) {
1140 if (udata && init_attr->create_flags & ~(sup_u_create_flags))
1141 return ERR_PTR(-EINVAL);
1142
1143 if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
1144 MLX4_IB_QP_CREATE_USE_GFP_NOIO |
1145 MLX4_IB_QP_CREATE_ROCE_V2_GSI |
1146 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) &&
1147 init_attr->qp_type != IB_QPT_UD) ||
1148 (init_attr->create_flags & MLX4_IB_SRIOV_SQP &&
1149 init_attr->qp_type > IB_QPT_GSI) ||
1150 (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
1151 init_attr->qp_type != IB_QPT_GSI))
1152 return ERR_PTR(-EINVAL);
1153 }
1154
1155 switch (init_attr->qp_type) {
1156 case IB_QPT_XRC_TGT:
1157 pd = to_mxrcd(init_attr->xrcd)->pd;
1158 xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1159 init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1160
1161 case IB_QPT_XRC_INI:
1162 if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1163 return ERR_PTR(-ENOSYS);
1164 init_attr->recv_cq = init_attr->send_cq;
1165
1166 case IB_QPT_RC:
1167 case IB_QPT_UC:
1168 case IB_QPT_RAW_PACKET:
1169 qp = kzalloc(sizeof *qp, gfp);
1170 if (!qp)
1171 return ERR_PTR(-ENOMEM);
1172 qp->pri.vid = 0xFFFF;
1173 qp->alt.vid = 0xFFFF;
1174
1175 case IB_QPT_UD:
1176 {
1177 err = create_qp_common(to_mdev(pd->device), pd, init_attr,
1178 udata, 0, &qp, gfp);
1179 if (err)
1180 return ERR_PTR(err);
1181
1182 qp->ibqp.qp_num = qp->mqp.qpn;
1183 qp->xrcdn = xrcdn;
1184
1185 break;
1186 }
1187 case IB_QPT_SMI:
1188 case IB_QPT_GSI:
1189 {
1190 int sqpn;
1191
1192
1193 if (udata)
1194 return ERR_PTR(-EINVAL);
1195 if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1196 int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev, 1, 1, &sqpn, 0);
1197
1198 if (res)
1199 return ERR_PTR(res);
1200 } else {
1201 sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
1202 }
1203
1204 err = create_qp_common(to_mdev(pd->device), pd, init_attr, udata,
1205 sqpn,
1206 &qp, gfp);
1207 if (err)
1208 return ERR_PTR(err);
1209
1210 qp->port = init_attr->port_num;
1211 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
1212 init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
1213 break;
1214 }
1215 default:
1216
1217 return ERR_PTR(-EINVAL);
1218 }
1219
1220 return &qp->ibqp;
1221}
1222
1223struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
1224 struct ib_qp_init_attr *init_attr,
1225 struct ib_udata *udata) {
1226 struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
1227 struct ib_qp *ibqp;
1228 struct mlx4_ib_dev *dev = to_mdev(device);
1229
1230 ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);
1231
1232 if (!IS_ERR(ibqp) &&
1233 (init_attr->qp_type == IB_QPT_GSI) &&
1234 !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
1235 struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));
1236 int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
1237
1238 if (is_eth &&
1239 dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
1240 init_attr->create_flags |= MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1241 sqp->roce_v2_gsi = ib_create_qp(pd, init_attr);
1242
1243 if (IS_ERR(sqp->roce_v2_gsi)) {
1244 pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
1245 sqp->roce_v2_gsi = NULL;
1246 } else {
1247 sqp = to_msqp(to_mqp(sqp->roce_v2_gsi));
1248 sqp->qp.flags |= MLX4_IB_ROCE_V2_GSI_QP;
1249 }
1250
1251 init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1252 }
1253 }
1254 return ibqp;
1255}
1256
1257static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
1258{
1259 struct mlx4_ib_dev *dev = to_mdev(qp->device);
1260 struct mlx4_ib_qp *mqp = to_mqp(qp);
1261 struct mlx4_ib_pd *pd;
1262
1263 if (is_qp0(dev, mqp))
1264 mlx4_CLOSE_PORT(dev->dev, mqp->port);
1265
1266 if (dev->qp1_proxy[mqp->port - 1] == mqp) {
1267 mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]);
1268 dev->qp1_proxy[mqp->port - 1] = NULL;
1269 mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]);
1270 }
1271
1272 if (mqp->counter_index)
1273 mlx4_ib_free_qp_counter(dev, mqp);
1274
1275 pd = get_pd(mqp);
1276 destroy_qp_common(dev, mqp, !!pd->ibpd.uobject);
1277
1278 if (is_sqp(dev, mqp))
1279 kfree(to_msqp(mqp));
1280 else
1281 kfree(mqp);
1282
1283 return 0;
1284}
1285
1286int mlx4_ib_destroy_qp(struct ib_qp *qp)
1287{
1288 struct mlx4_ib_qp *mqp = to_mqp(qp);
1289
1290 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
1291 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
1292
1293 if (sqp->roce_v2_gsi)
1294 ib_destroy_qp(sqp->roce_v2_gsi);
1295 }
1296
1297 return _mlx4_ib_destroy_qp(qp);
1298}
1299
1300static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
1301{
1302 switch (type) {
1303 case MLX4_IB_QPT_RC: return MLX4_QP_ST_RC;
1304 case MLX4_IB_QPT_UC: return MLX4_QP_ST_UC;
1305 case MLX4_IB_QPT_UD: return MLX4_QP_ST_UD;
1306 case MLX4_IB_QPT_XRC_INI:
1307 case MLX4_IB_QPT_XRC_TGT: return MLX4_QP_ST_XRC;
1308 case MLX4_IB_QPT_SMI:
1309 case MLX4_IB_QPT_GSI:
1310 case MLX4_IB_QPT_RAW_PACKET: return MLX4_QP_ST_MLX;
1311
1312 case MLX4_IB_QPT_PROXY_SMI_OWNER:
1313 case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ?
1314 MLX4_QP_ST_MLX : -1);
1315 case MLX4_IB_QPT_PROXY_SMI:
1316 case MLX4_IB_QPT_TUN_SMI:
1317 case MLX4_IB_QPT_PROXY_GSI:
1318 case MLX4_IB_QPT_TUN_GSI: return (mlx4_is_mfunc(dev->dev) ?
1319 MLX4_QP_ST_UD : -1);
1320 default: return -1;
1321 }
1322}
1323
1324static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
1325 int attr_mask)
1326{
1327 u8 dest_rd_atomic;
1328 u32 access_flags;
1329 u32 hw_access_flags = 0;
1330
1331 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1332 dest_rd_atomic = attr->max_dest_rd_atomic;
1333 else
1334 dest_rd_atomic = qp->resp_depth;
1335
1336 if (attr_mask & IB_QP_ACCESS_FLAGS)
1337 access_flags = attr->qp_access_flags;
1338 else
1339 access_flags = qp->atomic_rd_en;
1340
1341 if (!dest_rd_atomic)
1342 access_flags &= IB_ACCESS_REMOTE_WRITE;
1343
1344 if (access_flags & IB_ACCESS_REMOTE_READ)
1345 hw_access_flags |= MLX4_QP_BIT_RRE;
1346 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
1347 hw_access_flags |= MLX4_QP_BIT_RAE;
1348 if (access_flags & IB_ACCESS_REMOTE_WRITE)
1349 hw_access_flags |= MLX4_QP_BIT_RWE;
1350
1351 return cpu_to_be32(hw_access_flags);
1352}
1353
1354static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
1355 int attr_mask)
1356{
1357 if (attr_mask & IB_QP_PKEY_INDEX)
1358 sqp->pkey_index = attr->pkey_index;
1359 if (attr_mask & IB_QP_QKEY)
1360 sqp->qkey = attr->qkey;
1361 if (attr_mask & IB_QP_SQ_PSN)
1362 sqp->send_psn = attr->sq_psn;
1363}
1364
1365static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
1366{
1367 path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
1368}
1369
1370static int _mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
1371 u64 smac, u16 vlan_tag, struct mlx4_qp_path *path,
1372 struct mlx4_roce_smac_vlan_info *smac_info, u8 port)
1373{
1374 int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) ==
1375 IB_LINK_LAYER_ETHERNET;
1376 int vidx;
1377 int smac_index;
1378 int err;
1379
1380
1381 path->grh_mylmc = ah->src_path_bits & 0x7f;
1382 path->rlid = cpu_to_be16(ah->dlid);
1383 if (ah->static_rate) {
1384 path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET;
1385 while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
1386 !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
1387 --path->static_rate;
1388 } else
1389 path->static_rate = 0;
1390
1391 if (ah->ah_flags & IB_AH_GRH) {
1392 int real_sgid_index = mlx4_ib_gid_index_to_real_index(dev,
1393 port,
1394 ah->grh.sgid_index);
1395
1396 if (real_sgid_index >= dev->dev->caps.gid_table_len[port]) {
1397 pr_err("sgid_index (%u) too large. max is %d\n",
1398 real_sgid_index, dev->dev->caps.gid_table_len[port] - 1);
1399 return -1;
1400 }
1401
1402 path->grh_mylmc |= 1 << 7;
1403 path->mgid_index = real_sgid_index;
1404 path->hop_limit = ah->grh.hop_limit;
1405 path->tclass_flowlabel =
1406 cpu_to_be32((ah->grh.traffic_class << 20) |
1407 (ah->grh.flow_label));
1408 memcpy(path->rgid, ah->grh.dgid.raw, 16);
1409 }
1410
1411 if (is_eth) {
1412 if (!(ah->ah_flags & IB_AH_GRH))
1413 return -1;
1414
1415 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1416 ((port - 1) << 6) | ((ah->sl & 7) << 3);
1417
1418 path->feup |= MLX4_FEUP_FORCE_ETH_UP;
1419 if (vlan_tag < 0x1000) {
1420 if (smac_info->vid < 0x1000) {
1421
1422 if (smac_info->vid != vlan_tag) {
1423
1424 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1425 if (err)
1426 return err;
1427 smac_info->candidate_vid = vlan_tag;
1428 smac_info->candidate_vlan_index = vidx;
1429 smac_info->candidate_vlan_port = port;
1430 smac_info->update_vid = 1;
1431 path->vlan_index = vidx;
1432 } else {
1433 path->vlan_index = smac_info->vlan_index;
1434 }
1435 } else {
1436
1437 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1438 if (err)
1439 return err;
1440 smac_info->candidate_vid = vlan_tag;
1441 smac_info->candidate_vlan_index = vidx;
1442 smac_info->candidate_vlan_port = port;
1443 smac_info->update_vid = 1;
1444 path->vlan_index = vidx;
1445 }
1446 path->feup |= MLX4_FVL_FORCE_ETH_VLAN;
1447 path->fl = 1 << 6;
1448 } else {
1449
1450 if (smac_info->vid < 0x1000) {
1451 smac_info->candidate_vid = 0xFFFF;
1452 smac_info->update_vid = 1;
1453 }
1454 }
1455
1456
1457
1458
1459
1460
1461 if ((!smac_info->smac && !smac_info->smac_port) ||
1462 smac_info->smac != smac) {
1463
1464 smac_index = mlx4_register_mac(dev->dev, port, smac);
1465 if (smac_index >= 0) {
1466 smac_info->candidate_smac_index = smac_index;
1467 smac_info->candidate_smac = smac;
1468 smac_info->candidate_smac_port = port;
1469 } else {
1470 return -EINVAL;
1471 }
1472 } else {
1473 smac_index = smac_info->smac_index;
1474 }
1475
1476 memcpy(path->dmac, ah->dmac, 6);
1477 path->ackto = MLX4_IB_LINK_TYPE_ETH;
1478
1479 path->grh_mylmc = (u8) (smac_index) | 0x80;
1480 } else {
1481 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1482 ((port - 1) << 6) | ((ah->sl & 0xf) << 2);
1483 }
1484
1485 return 0;
1486}
1487
1488static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp,
1489 enum ib_qp_attr_mask qp_attr_mask,
1490 struct mlx4_ib_qp *mqp,
1491 struct mlx4_qp_path *path, u8 port,
1492 u16 vlan_id, u8 *smac)
1493{
1494 return _mlx4_set_path(dev, &qp->ah_attr,
1495 mlx4_mac_to_u64(smac),
1496 vlan_id,
1497 path, &mqp->pri, port);
1498}
1499
1500static int mlx4_set_alt_path(struct mlx4_ib_dev *dev,
1501 const struct ib_qp_attr *qp,
1502 enum ib_qp_attr_mask qp_attr_mask,
1503 struct mlx4_ib_qp *mqp,
1504 struct mlx4_qp_path *path, u8 port)
1505{
1506 return _mlx4_set_path(dev, &qp->alt_ah_attr,
1507 0,
1508 0xffff,
1509 path, &mqp->alt, port);
1510}
1511
1512static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1513{
1514 struct mlx4_ib_gid_entry *ge, *tmp;
1515
1516 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1517 if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
1518 ge->added = 1;
1519 ge->port = qp->port;
1520 }
1521 }
1522}
1523
1524static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev,
1525 struct mlx4_ib_qp *qp,
1526 struct mlx4_qp_context *context)
1527{
1528 u64 u64_mac;
1529 int smac_index;
1530
1531 u64_mac = atomic64_read(&dev->iboe.mac[qp->port - 1]);
1532
1533 context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6);
1534 if (!qp->pri.smac && !qp->pri.smac_port) {
1535 smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac);
1536 if (smac_index >= 0) {
1537 qp->pri.candidate_smac_index = smac_index;
1538 qp->pri.candidate_smac = u64_mac;
1539 qp->pri.candidate_smac_port = qp->port;
1540 context->pri_path.grh_mylmc = 0x80 | (u8) smac_index;
1541 } else {
1542 return -ENOENT;
1543 }
1544 }
1545 return 0;
1546}
1547
1548static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1549{
1550 struct counter_index *new_counter_index;
1551 int err;
1552 u32 tmp_idx;
1553
1554 if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) !=
1555 IB_LINK_LAYER_ETHERNET ||
1556 !(qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) ||
1557 !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
1558 return 0;
1559
1560 err = mlx4_counter_alloc(dev->dev, &tmp_idx);
1561 if (err)
1562 return err;
1563
1564 new_counter_index = kmalloc(sizeof(*new_counter_index), GFP_KERNEL);
1565 if (!new_counter_index) {
1566 mlx4_counter_free(dev->dev, tmp_idx);
1567 return -ENOMEM;
1568 }
1569
1570 new_counter_index->index = tmp_idx;
1571 new_counter_index->allocated = 1;
1572 qp->counter_index = new_counter_index;
1573
1574 mutex_lock(&dev->counters_table[qp->port - 1].mutex);
1575 list_add_tail(&new_counter_index->list,
1576 &dev->counters_table[qp->port - 1].counters_list);
1577 mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
1578
1579 return 0;
1580}
1581
1582enum {
1583 MLX4_QPC_ROCE_MODE_1 = 0,
1584 MLX4_QPC_ROCE_MODE_2 = 2,
1585 MLX4_QPC_ROCE_MODE_UNDEFINED = 0xff
1586};
1587
1588static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
1589{
1590 switch (gid_type) {
1591 case IB_GID_TYPE_ROCE:
1592 return MLX4_QPC_ROCE_MODE_1;
1593 case IB_GID_TYPE_ROCE_UDP_ENCAP:
1594 return MLX4_QPC_ROCE_MODE_2;
1595 default:
1596 return MLX4_QPC_ROCE_MODE_UNDEFINED;
1597 }
1598}
1599
1600static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
1601 const struct ib_qp_attr *attr, int attr_mask,
1602 enum ib_qp_state cur_state, enum ib_qp_state new_state)
1603{
1604 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
1605 struct mlx4_ib_qp *qp = to_mqp(ibqp);
1606 struct mlx4_ib_pd *pd;
1607 struct mlx4_ib_cq *send_cq, *recv_cq;
1608 struct mlx4_qp_context *context;
1609 enum mlx4_qp_optpar optpar = 0;
1610 int sqd_event;
1611 int steer_qp = 0;
1612 int err = -EINVAL;
1613 int counter_index;
1614
1615
1616 if (attr_mask & IB_QP_ALT_PATH &&
1617 rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
1618 IB_LINK_LAYER_ETHERNET)
1619 return -ENOTSUPP;
1620
1621 context = kzalloc(sizeof *context, GFP_KERNEL);
1622 if (!context)
1623 return -ENOMEM;
1624
1625 context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
1626 (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16));
1627
1628 if (!(attr_mask & IB_QP_PATH_MIG_STATE))
1629 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
1630 else {
1631 optpar |= MLX4_QP_OPTPAR_PM_STATE;
1632 switch (attr->path_mig_state) {
1633 case IB_MIG_MIGRATED:
1634 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
1635 break;
1636 case IB_MIG_REARM:
1637 context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
1638 break;
1639 case IB_MIG_ARMED:
1640 context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
1641 break;
1642 }
1643 }
1644
1645 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
1646 context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
1647 else if (ibqp->qp_type == IB_QPT_RAW_PACKET)
1648 context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
1649 else if (ibqp->qp_type == IB_QPT_UD) {
1650 if (qp->flags & MLX4_IB_QP_LSO)
1651 context->mtu_msgmax = (IB_MTU_4096 << 5) |
1652 ilog2(dev->dev->caps.max_gso_sz);
1653 else
1654 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
1655 } else if (attr_mask & IB_QP_PATH_MTU) {
1656 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
1657 pr_err("path MTU (%u) is invalid\n",
1658 attr->path_mtu);
1659 goto out;
1660 }
1661 context->mtu_msgmax = (attr->path_mtu << 5) |
1662 ilog2(dev->dev->caps.max_msg_sz);
1663 }
1664
1665 if (qp->rq.wqe_cnt)
1666 context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
1667 context->rq_size_stride |= qp->rq.wqe_shift - 4;
1668
1669 if (qp->sq.wqe_cnt)
1670 context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
1671 context->sq_size_stride |= qp->sq.wqe_shift - 4;
1672
1673 if (new_state == IB_QPS_RESET && qp->counter_index)
1674 mlx4_ib_free_qp_counter(dev, qp);
1675
1676 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1677 context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
1678 context->xrcd = cpu_to_be32((u32) qp->xrcdn);
1679 if (ibqp->qp_type == IB_QPT_RAW_PACKET)
1680 context->param3 |= cpu_to_be32(1 << 30);
1681 }
1682
1683 if (qp->ibqp.uobject)
1684 context->usr_page = cpu_to_be32(
1685 mlx4_to_hw_uar_index(dev->dev,
1686 to_mucontext(ibqp->uobject->context)->uar.index));
1687 else
1688 context->usr_page = cpu_to_be32(
1689 mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
1690
1691 if (attr_mask & IB_QP_DEST_QPN)
1692 context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
1693
1694 if (attr_mask & IB_QP_PORT) {
1695 if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
1696 !(attr_mask & IB_QP_AV)) {
1697 mlx4_set_sched(&context->pri_path, attr->port_num);
1698 optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
1699 }
1700 }
1701
1702 if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
1703 err = create_qp_lb_counter(dev, qp);
1704 if (err)
1705 goto out;
1706
1707 counter_index =
1708 dev->counters_table[qp->port - 1].default_counter;
1709 if (qp->counter_index)
1710 counter_index = qp->counter_index->index;
1711
1712 if (counter_index != -1) {
1713 context->pri_path.counter_index = counter_index;
1714 optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX;
1715 if (qp->counter_index) {
1716 context->pri_path.fl |=
1717 MLX4_FL_ETH_SRC_CHECK_MC_LB;
1718 context->pri_path.vlan_control |=
1719 MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER;
1720 }
1721 } else
1722 context->pri_path.counter_index =
1723 MLX4_SINK_COUNTER_INDEX(dev->dev);
1724
1725 if (qp->flags & MLX4_IB_QP_NETIF) {
1726 mlx4_ib_steer_qp_reg(dev, qp, 1);
1727 steer_qp = 1;
1728 }
1729
1730 if (ibqp->qp_type == IB_QPT_GSI) {
1731 enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ?
1732 IB_GID_TYPE_ROCE_UDP_ENCAP : IB_GID_TYPE_ROCE;
1733 u8 qpc_roce_mode = gid_type_to_qpc(gid_type);
1734
1735 context->rlkey_roce_mode |= (qpc_roce_mode << 6);
1736 }
1737 }
1738
1739 if (attr_mask & IB_QP_PKEY_INDEX) {
1740 if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
1741 context->pri_path.disable_pkey_check = 0x40;
1742 context->pri_path.pkey_index = attr->pkey_index;
1743 optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
1744 }
1745
1746 if (attr_mask & IB_QP_AV) {
1747 u8 port_num = mlx4_is_bonded(to_mdev(ibqp->device)->dev) ? 1 :
1748 attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1749 union ib_gid gid;
1750 struct ib_gid_attr gid_attr;
1751 u16 vlan = 0xffff;
1752 u8 smac[ETH_ALEN];
1753 int status = 0;
1754 int is_eth = rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
1755 attr->ah_attr.ah_flags & IB_AH_GRH;
1756
1757 if (is_eth) {
1758 int index = attr->ah_attr.grh.sgid_index;
1759
1760 status = ib_get_cached_gid(ibqp->device, port_num,
1761 index, &gid, &gid_attr);
1762 if (!status && !memcmp(&gid, &zgid, sizeof(gid)))
1763 status = -ENOENT;
1764 if (!status && gid_attr.ndev) {
1765 vlan = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1766 memcpy(smac, gid_attr.ndev->dev_addr, ETH_ALEN);
1767 dev_put(gid_attr.ndev);
1768 }
1769 }
1770 if (status)
1771 goto out;
1772
1773 if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
1774 port_num, vlan, smac))
1775 goto out;
1776
1777 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
1778 MLX4_QP_OPTPAR_SCHED_QUEUE);
1779
1780 if (is_eth &&
1781 (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)) {
1782 u8 qpc_roce_mode = gid_type_to_qpc(gid_attr.gid_type);
1783
1784 if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_UNDEFINED) {
1785 err = -EINVAL;
1786 goto out;
1787 }
1788 context->rlkey_roce_mode |= (qpc_roce_mode << 6);
1789 }
1790
1791 }
1792
1793 if (attr_mask & IB_QP_TIMEOUT) {
1794 context->pri_path.ackto |= attr->timeout << 3;
1795 optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
1796 }
1797
1798 if (attr_mask & IB_QP_ALT_PATH) {
1799 if (attr->alt_port_num == 0 ||
1800 attr->alt_port_num > dev->dev->caps.num_ports)
1801 goto out;
1802
1803 if (attr->alt_pkey_index >=
1804 dev->dev->caps.pkey_table_len[attr->alt_port_num])
1805 goto out;
1806
1807 if (mlx4_set_alt_path(dev, attr, attr_mask, qp,
1808 &context->alt_path,
1809 attr->alt_port_num))
1810 goto out;
1811
1812 context->alt_path.pkey_index = attr->alt_pkey_index;
1813 context->alt_path.ackto = attr->alt_timeout << 3;
1814 optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
1815 }
1816
1817 pd = get_pd(qp);
1818 get_cqs(qp, &send_cq, &recv_cq);
1819 context->pd = cpu_to_be32(pd->pdn);
1820 context->cqn_send = cpu_to_be32(send_cq->mcq.cqn);
1821 context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
1822 context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
1823
1824
1825 if (!qp->ibqp.uobject)
1826 context->params1 |= cpu_to_be32(1 << 11);
1827
1828 if (attr_mask & IB_QP_RNR_RETRY) {
1829 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
1830 optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
1831 }
1832
1833 if (attr_mask & IB_QP_RETRY_CNT) {
1834 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
1835 optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
1836 }
1837
1838 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1839 if (attr->max_rd_atomic)
1840 context->params1 |=
1841 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
1842 optpar |= MLX4_QP_OPTPAR_SRA_MAX;
1843 }
1844
1845 if (attr_mask & IB_QP_SQ_PSN)
1846 context->next_send_psn = cpu_to_be32(attr->sq_psn);
1847
1848 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1849 if (attr->max_dest_rd_atomic)
1850 context->params2 |=
1851 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
1852 optpar |= MLX4_QP_OPTPAR_RRA_MAX;
1853 }
1854
1855 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
1856 context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
1857 optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
1858 }
1859
1860 if (ibqp->srq)
1861 context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
1862
1863 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1864 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
1865 optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
1866 }
1867 if (attr_mask & IB_QP_RQ_PSN)
1868 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
1869
1870
1871 if (attr_mask & IB_QP_QKEY) {
1872 if (qp->mlx4_ib_qp_type &
1873 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))
1874 context->qkey = cpu_to_be32(IB_QP_SET_QKEY);
1875 else {
1876 if (mlx4_is_mfunc(dev->dev) &&
1877 !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) &&
1878 (attr->qkey & MLX4_RESERVED_QKEY_MASK) ==
1879 MLX4_RESERVED_QKEY_BASE) {
1880 pr_err("Cannot use reserved QKEY"
1881 " 0x%x (range 0xffff0000..0xffffffff"
1882 " is reserved)\n", attr->qkey);
1883 err = -EINVAL;
1884 goto out;
1885 }
1886 context->qkey = cpu_to_be32(attr->qkey);
1887 }
1888 optpar |= MLX4_QP_OPTPAR_Q_KEY;
1889 }
1890
1891 if (ibqp->srq)
1892 context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
1893
1894 if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1895 context->db_rec_addr = cpu_to_be64(qp->db.dma);
1896
1897 if (cur_state == IB_QPS_INIT &&
1898 new_state == IB_QPS_RTR &&
1899 (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
1900 ibqp->qp_type == IB_QPT_UD ||
1901 ibqp->qp_type == IB_QPT_RAW_PACKET)) {
1902 context->pri_path.sched_queue = (qp->port - 1) << 6;
1903 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
1904 qp->mlx4_ib_qp_type &
1905 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) {
1906 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
1907 if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI)
1908 context->pri_path.fl = 0x80;
1909 } else {
1910 if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
1911 context->pri_path.fl = 0x80;
1912 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
1913 }
1914 if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
1915 IB_LINK_LAYER_ETHERNET) {
1916 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI ||
1917 qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI)
1918 context->pri_path.feup = 1 << 7;
1919
1920 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD ||
1921 qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI ||
1922 qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) {
1923 err = handle_eth_ud_smac_index(dev, qp, context);
1924 if (err) {
1925 err = -EINVAL;
1926 goto out;
1927 }
1928 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1929 dev->qp1_proxy[qp->port - 1] = qp;
1930 }
1931 }
1932 }
1933
1934 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
1935 context->pri_path.ackto = (context->pri_path.ackto & 0xf8) |
1936 MLX4_IB_LINK_TYPE_ETH;
1937 if (dev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
1938
1939 if (!(context->flags & cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET)))
1940 context->srqn = cpu_to_be32(7 << 28);
1941 }
1942 }
1943
1944 if (ibqp->qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
1945 int is_eth = rdma_port_get_link_layer(
1946 &dev->ib_dev, qp->port) ==
1947 IB_LINK_LAYER_ETHERNET;
1948 if (is_eth) {
1949 context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH;
1950 optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH;
1951 }
1952 }
1953
1954
1955 if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD &&
1956 attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
1957 sqd_event = 1;
1958 else
1959 sqd_event = 0;
1960
1961 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1962 context->rlkey_roce_mode |= (1 << 4);
1963
1964
1965
1966
1967
1968
1969
1970 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1971 struct mlx4_wqe_ctrl_seg *ctrl;
1972 int i;
1973
1974 for (i = 0; i < qp->sq.wqe_cnt; ++i) {
1975 ctrl = get_send_wqe(qp, i);
1976 ctrl->owner_opcode = cpu_to_be32(1 << 31);
1977 if (qp->sq_max_wqes_per_wr == 1)
1978 ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4);
1979
1980 stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
1981 }
1982 }
1983
1984 err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
1985 to_mlx4_state(new_state), context, optpar,
1986 sqd_event, &qp->mqp);
1987 if (err)
1988 goto out;
1989
1990 qp->state = new_state;
1991
1992 if (attr_mask & IB_QP_ACCESS_FLAGS)
1993 qp->atomic_rd_en = attr->qp_access_flags;
1994 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1995 qp->resp_depth = attr->max_dest_rd_atomic;
1996 if (attr_mask & IB_QP_PORT) {
1997 qp->port = attr->port_num;
1998 update_mcg_macs(dev, qp);
1999 }
2000 if (attr_mask & IB_QP_ALT_PATH)
2001 qp->alt_port = attr->alt_port_num;
2002
2003 if (is_sqp(dev, qp))
2004 store_sqp_attrs(to_msqp(qp), attr, attr_mask);
2005
2006
2007
2008
2009
2010 if (is_qp0(dev, qp)) {
2011 if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
2012 if (mlx4_INIT_PORT(dev->dev, qp->port))
2013 pr_warn("INIT_PORT failed for port %d\n",
2014 qp->port);
2015
2016 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
2017 (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
2018 mlx4_CLOSE_PORT(dev->dev, qp->port);
2019 }
2020
2021
2022
2023
2024
2025 if (new_state == IB_QPS_RESET) {
2026 if (!ibqp->uobject) {
2027 mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
2028 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
2029 if (send_cq != recv_cq)
2030 mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
2031
2032 qp->rq.head = 0;
2033 qp->rq.tail = 0;
2034 qp->sq.head = 0;
2035 qp->sq.tail = 0;
2036 qp->sq_next_wqe = 0;
2037 if (qp->rq.wqe_cnt)
2038 *qp->db.db = 0;
2039
2040 if (qp->flags & MLX4_IB_QP_NETIF)
2041 mlx4_ib_steer_qp_reg(dev, qp, 0);
2042 }
2043 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
2044 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2045 qp->pri.smac = 0;
2046 qp->pri.smac_port = 0;
2047 }
2048 if (qp->alt.smac) {
2049 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2050 qp->alt.smac = 0;
2051 }
2052 if (qp->pri.vid < 0x1000) {
2053 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
2054 qp->pri.vid = 0xFFFF;
2055 qp->pri.candidate_vid = 0xFFFF;
2056 qp->pri.update_vid = 0;
2057 }
2058
2059 if (qp->alt.vid < 0x1000) {
2060 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
2061 qp->alt.vid = 0xFFFF;
2062 qp->alt.candidate_vid = 0xFFFF;
2063 qp->alt.update_vid = 0;
2064 }
2065 }
2066out:
2067 if (err && qp->counter_index)
2068 mlx4_ib_free_qp_counter(dev, qp);
2069 if (err && steer_qp)
2070 mlx4_ib_steer_qp_reg(dev, qp, 0);
2071 kfree(context);
2072 if (qp->pri.candidate_smac ||
2073 (!qp->pri.candidate_smac && qp->pri.candidate_smac_port)) {
2074 if (err) {
2075 mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac);
2076 } else {
2077 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port))
2078 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2079 qp->pri.smac = qp->pri.candidate_smac;
2080 qp->pri.smac_index = qp->pri.candidate_smac_index;
2081 qp->pri.smac_port = qp->pri.candidate_smac_port;
2082 }
2083 qp->pri.candidate_smac = 0;
2084 qp->pri.candidate_smac_index = 0;
2085 qp->pri.candidate_smac_port = 0;
2086 }
2087 if (qp->alt.candidate_smac) {
2088 if (err) {
2089 mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->alt.candidate_smac);
2090 } else {
2091 if (qp->alt.smac)
2092 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2093 qp->alt.smac = qp->alt.candidate_smac;
2094 qp->alt.smac_index = qp->alt.candidate_smac_index;
2095 qp->alt.smac_port = qp->alt.candidate_smac_port;
2096 }
2097 qp->alt.candidate_smac = 0;
2098 qp->alt.candidate_smac_index = 0;
2099 qp->alt.candidate_smac_port = 0;
2100 }
2101
2102 if (qp->pri.update_vid) {
2103 if (err) {
2104 if (qp->pri.candidate_vid < 0x1000)
2105 mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port,
2106 qp->pri.candidate_vid);
2107 } else {
2108 if (qp->pri.vid < 0x1000)
2109 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port,
2110 qp->pri.vid);
2111 qp->pri.vid = qp->pri.candidate_vid;
2112 qp->pri.vlan_port = qp->pri.candidate_vlan_port;
2113 qp->pri.vlan_index = qp->pri.candidate_vlan_index;
2114 }
2115 qp->pri.candidate_vid = 0xFFFF;
2116 qp->pri.update_vid = 0;
2117 }
2118
2119 if (qp->alt.update_vid) {
2120 if (err) {
2121 if (qp->alt.candidate_vid < 0x1000)
2122 mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port,
2123 qp->alt.candidate_vid);
2124 } else {
2125 if (qp->alt.vid < 0x1000)
2126 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port,
2127 qp->alt.vid);
2128 qp->alt.vid = qp->alt.candidate_vid;
2129 qp->alt.vlan_port = qp->alt.candidate_vlan_port;
2130 qp->alt.vlan_index = qp->alt.candidate_vlan_index;
2131 }
2132 qp->alt.candidate_vid = 0xFFFF;
2133 qp->alt.update_vid = 0;
2134 }
2135
2136 return err;
2137}
2138
2139static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2140 int attr_mask, struct ib_udata *udata)
2141{
2142 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2143 struct mlx4_ib_qp *qp = to_mqp(ibqp);
2144 enum ib_qp_state cur_state, new_state;
2145 int err = -EINVAL;
2146 int ll;
2147 mutex_lock(&qp->mutex);
2148
2149 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2150 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2151
2152 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
2153 ll = IB_LINK_LAYER_UNSPECIFIED;
2154 } else {
2155 int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2156 ll = rdma_port_get_link_layer(&dev->ib_dev, port);
2157 }
2158
2159 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
2160 attr_mask, ll)) {
2161 pr_debug("qpn 0x%x: invalid attribute mask specified "
2162 "for transition %d to %d. qp_type %d,"
2163 " attr_mask 0x%x\n",
2164 ibqp->qp_num, cur_state, new_state,
2165 ibqp->qp_type, attr_mask);
2166 goto out;
2167 }
2168
2169 if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT)) {
2170 if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
2171 if ((ibqp->qp_type == IB_QPT_RC) ||
2172 (ibqp->qp_type == IB_QPT_UD) ||
2173 (ibqp->qp_type == IB_QPT_UC) ||
2174 (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
2175 (ibqp->qp_type == IB_QPT_XRC_INI)) {
2176 attr->port_num = mlx4_ib_bond_next_port(dev);
2177 }
2178 } else {
2179
2180
2181 attr_mask &= ~IB_QP_PORT;
2182 }
2183 }
2184
2185 if ((attr_mask & IB_QP_PORT) &&
2186 (attr->port_num == 0 || attr->port_num > dev->num_ports)) {
2187 pr_debug("qpn 0x%x: invalid port number (%d) specified "
2188 "for transition %d to %d. qp_type %d\n",
2189 ibqp->qp_num, attr->port_num, cur_state,
2190 new_state, ibqp->qp_type);
2191 goto out;
2192 }
2193
2194 if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
2195 (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) !=
2196 IB_LINK_LAYER_ETHERNET))
2197 goto out;
2198
2199 if (attr_mask & IB_QP_PKEY_INDEX) {
2200 int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2201 if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) {
2202 pr_debug("qpn 0x%x: invalid pkey index (%d) specified "
2203 "for transition %d to %d. qp_type %d\n",
2204 ibqp->qp_num, attr->pkey_index, cur_state,
2205 new_state, ibqp->qp_type);
2206 goto out;
2207 }
2208 }
2209
2210 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
2211 attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
2212 pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. "
2213 "Transition %d to %d. qp_type %d\n",
2214 ibqp->qp_num, attr->max_rd_atomic, cur_state,
2215 new_state, ibqp->qp_type);
2216 goto out;
2217 }
2218
2219 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
2220 attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
2221 pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. "
2222 "Transition %d to %d. qp_type %d\n",
2223 ibqp->qp_num, attr->max_dest_rd_atomic, cur_state,
2224 new_state, ibqp->qp_type);
2225 goto out;
2226 }
2227
2228 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
2229 err = 0;
2230 goto out;
2231 }
2232
2233 err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
2234
2235 if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
2236 attr->port_num = 1;
2237
2238out:
2239 mutex_unlock(&qp->mutex);
2240 return err;
2241}
2242
2243int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2244 int attr_mask, struct ib_udata *udata)
2245{
2246 struct mlx4_ib_qp *mqp = to_mqp(ibqp);
2247 int ret;
2248
2249 ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
2250
2251 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2252 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
2253 int err = 0;
2254
2255 if (sqp->roce_v2_gsi)
2256 err = ib_modify_qp(sqp->roce_v2_gsi, attr, attr_mask);
2257 if (err)
2258 pr_err("Failed to modify GSI QP for RoCEv2 (%d)\n",
2259 err);
2260 }
2261 return ret;
2262}
2263
2264static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
2265{
2266 int i;
2267 for (i = 0; i < dev->caps.num_ports; i++) {
2268 if (qpn == dev->caps.qp0_proxy[i] ||
2269 qpn == dev->caps.qp0_tunnel[i]) {
2270 *qkey = dev->caps.qp0_qkey[i];
2271 return 0;
2272 }
2273 }
2274 return -EINVAL;
2275}
2276
2277static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
2278 struct ib_ud_wr *wr,
2279 void *wqe, unsigned *mlx_seg_len)
2280{
2281 struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device);
2282 struct ib_device *ib_dev = &mdev->ib_dev;
2283 struct mlx4_wqe_mlx_seg *mlx = wqe;
2284 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2285 struct mlx4_ib_ah *ah = to_mah(wr->ah);
2286 u16 pkey;
2287 u32 qkey;
2288 int send_size;
2289 int header_size;
2290 int spc;
2291 int i;
2292
2293 if (wr->wr.opcode != IB_WR_SEND)
2294 return -EINVAL;
2295
2296 send_size = 0;
2297
2298 for (i = 0; i < wr->wr.num_sge; ++i)
2299 send_size += wr->wr.sg_list[i].length;
2300
2301
2302
2303 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2304 send_size += sizeof (struct mlx4_ib_tunnel_header);
2305
2306 ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
2307
2308 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2309 sqp->ud_header.lrh.service_level =
2310 be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2311 sqp->ud_header.lrh.destination_lid =
2312 cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2313 sqp->ud_header.lrh.source_lid =
2314 cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2315 }
2316
2317 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2318
2319
2320 mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR);
2321 mlx->rlid = sqp->ud_header.lrh.destination_lid;
2322
2323 sqp->ud_header.lrh.virtual_lane = 0;
2324 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2325 ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey);
2326 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2327 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2328 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2329 else
2330 sqp->ud_header.bth.destination_qpn =
2331 cpu_to_be32(mdev->dev->caps.qp0_tunnel[sqp->qp.port - 1]);
2332
2333 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2334 if (mlx4_is_master(mdev->dev)) {
2335 if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2336 return -EINVAL;
2337 } else {
2338 if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2339 return -EINVAL;
2340 }
2341 sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
2342 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
2343
2344 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2345 sqp->ud_header.immediate_present = 0;
2346
2347 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
2348
2349
2350
2351
2352
2353
2354
2355 spc = MLX4_INLINE_ALIGN -
2356 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2357 if (header_size <= spc) {
2358 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
2359 memcpy(inl + 1, sqp->header_buf, header_size);
2360 i = 1;
2361 } else {
2362 inl->byte_count = cpu_to_be32(1 << 31 | spc);
2363 memcpy(inl + 1, sqp->header_buf, spc);
2364
2365 inl = (void *) (inl + 1) + spc;
2366 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380 wmb();
2381 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
2382 i = 2;
2383 }
2384
2385 *mlx_seg_len =
2386 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
2387 return 0;
2388}
2389
2390#define MLX4_ROCEV2_QP1_SPORT 0xC000
2391static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
2392 void *wqe, unsigned *mlx_seg_len)
2393{
2394 struct ib_device *ib_dev = sqp->qp.ibqp.device;
2395 struct mlx4_wqe_mlx_seg *mlx = wqe;
2396 struct mlx4_wqe_ctrl_seg *ctrl = wqe;
2397 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2398 struct mlx4_ib_ah *ah = to_mah(wr->ah);
2399 union ib_gid sgid;
2400 u16 pkey;
2401 int send_size;
2402 int header_size;
2403 int spc;
2404 int i;
2405 int err = 0;
2406 u16 vlan = 0xffff;
2407 bool is_eth;
2408 bool is_vlan = false;
2409 bool is_grh;
2410 bool is_udp = false;
2411 int ip_version = 0;
2412
2413 send_size = 0;
2414 for (i = 0; i < wr->wr.num_sge; ++i)
2415 send_size += wr->wr.sg_list[i].length;
2416
2417 is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
2418 is_grh = mlx4_ib_ah_grh_present(ah);
2419 if (is_eth) {
2420 struct ib_gid_attr gid_attr;
2421
2422 if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
2423
2424
2425
2426 err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev,
2427 be32_to_cpu(ah->av.ib.port_pd) >> 24,
2428 ah->av.ib.gid_index, &sgid.raw[0]);
2429 if (err)
2430 return err;
2431 } else {
2432 err = ib_get_cached_gid(ib_dev,
2433 be32_to_cpu(ah->av.ib.port_pd) >> 24,
2434 ah->av.ib.gid_index, &sgid,
2435 &gid_attr);
2436 if (!err) {
2437 if (gid_attr.ndev)
2438 dev_put(gid_attr.ndev);
2439 if (!memcmp(&sgid, &zgid, sizeof(sgid)))
2440 err = -ENOENT;
2441 }
2442 if (!err) {
2443 is_udp = gid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2444 if (is_udp) {
2445 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid))
2446 ip_version = 4;
2447 else
2448 ip_version = 6;
2449 is_grh = false;
2450 }
2451 } else {
2452 return err;
2453 }
2454 }
2455 if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
2456 vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
2457 is_vlan = 1;
2458 }
2459 }
2460 err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
2461 ip_version, is_udp, 0, &sqp->ud_header);
2462 if (err)
2463 return err;
2464
2465 if (!is_eth) {
2466 sqp->ud_header.lrh.service_level =
2467 be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2468 sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
2469 sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2470 }
2471
2472 if (is_grh || (ip_version == 6)) {
2473 sqp->ud_header.grh.traffic_class =
2474 (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
2475 sqp->ud_header.grh.flow_label =
2476 ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
2477 sqp->ud_header.grh.hop_limit = ah->av.ib.hop_limit;
2478 if (is_eth)
2479 memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16);
2480 else {
2481 if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
2482
2483
2484
2485 sqp->ud_header.grh.source_gid.global.subnet_prefix =
2486 to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
2487 subnet_prefix;
2488 sqp->ud_header.grh.source_gid.global.interface_id =
2489 to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
2490 guid_cache[ah->av.ib.gid_index];
2491 } else
2492 ib_get_cached_gid(ib_dev,
2493 be32_to_cpu(ah->av.ib.port_pd) >> 24,
2494 ah->av.ib.gid_index,
2495 &sqp->ud_header.grh.source_gid, NULL);
2496 }
2497 memcpy(sqp->ud_header.grh.destination_gid.raw,
2498 ah->av.ib.dgid, 16);
2499 }
2500
2501 if (ip_version == 4) {
2502 sqp->ud_header.ip4.tos =
2503 (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
2504 sqp->ud_header.ip4.id = 0;
2505 sqp->ud_header.ip4.frag_off = htons(IP_DF);
2506 sqp->ud_header.ip4.ttl = ah->av.eth.hop_limit;
2507
2508 memcpy(&sqp->ud_header.ip4.saddr,
2509 sgid.raw + 12, 4);
2510 memcpy(&sqp->ud_header.ip4.daddr, ah->av.ib.dgid + 12, 4);
2511 sqp->ud_header.ip4.check = ib_ud_ip4_csum(&sqp->ud_header);
2512 }
2513
2514 if (is_udp) {
2515 sqp->ud_header.udp.dport = htons(ROCE_V2_UDP_DPORT);
2516 sqp->ud_header.udp.sport = htons(MLX4_ROCEV2_QP1_SPORT);
2517 sqp->ud_header.udp.csum = 0;
2518 }
2519
2520 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2521
2522 if (!is_eth) {
2523 mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
2524 (sqp->ud_header.lrh.destination_lid ==
2525 IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
2526 (sqp->ud_header.lrh.service_level << 8));
2527 if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
2528 mlx->flags |= cpu_to_be32(0x1);
2529 mlx->rlid = sqp->ud_header.lrh.destination_lid;
2530 }
2531
2532 switch (wr->wr.opcode) {
2533 case IB_WR_SEND:
2534 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2535 sqp->ud_header.immediate_present = 0;
2536 break;
2537 case IB_WR_SEND_WITH_IMM:
2538 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2539 sqp->ud_header.immediate_present = 1;
2540 sqp->ud_header.immediate_data = wr->wr.ex.imm_data;
2541 break;
2542 default:
2543 return -EINVAL;
2544 }
2545
2546 if (is_eth) {
2547 struct in6_addr in6;
2548 u16 ether_type;
2549 u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
2550
2551 ether_type = (!is_udp) ? MLX4_IB_IBOE_ETHERTYPE :
2552 (ip_version == 4 ? ETH_P_IP : ETH_P_IPV6);
2553
2554 mlx->sched_prio = cpu_to_be16(pcp);
2555
2556 ether_addr_copy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac);
2557 memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
2558 memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2);
2559 memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4);
2560 memcpy(&in6, sgid.raw, sizeof(in6));
2561
2562
2563 if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
2564 mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
2565 if (!is_vlan) {
2566 sqp->ud_header.eth.type = cpu_to_be16(ether_type);
2567 } else {
2568 sqp->ud_header.vlan.type = cpu_to_be16(ether_type);
2569 sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
2570 }
2571 } else {
2572 sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0;
2573 if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
2574 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
2575 }
2576 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2577 if (!sqp->qp.ibqp.qp_num)
2578 ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
2579 else
2580 ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index, &pkey);
2581 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2582 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2583 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2584 sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ?
2585 sqp->qkey : wr->remote_qkey);
2586 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
2587
2588 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
2589
2590 if (0) {
2591 pr_err("built UD header of size %d:\n", header_size);
2592 for (i = 0; i < header_size / 4; ++i) {
2593 if (i % 8 == 0)
2594 pr_err(" [%02x] ", i * 4);
2595 pr_cont(" %08x",
2596 be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
2597 if ((i + 1) % 8 == 0)
2598 pr_cont("\n");
2599 }
2600 pr_err("\n");
2601 }
2602
2603
2604
2605
2606
2607
2608
2609 spc = MLX4_INLINE_ALIGN -
2610 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2611 if (header_size <= spc) {
2612 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
2613 memcpy(inl + 1, sqp->header_buf, header_size);
2614 i = 1;
2615 } else {
2616 inl->byte_count = cpu_to_be32(1 << 31 | spc);
2617 memcpy(inl + 1, sqp->header_buf, spc);
2618
2619 inl = (void *) (inl + 1) + spc;
2620 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634 wmb();
2635 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
2636 i = 2;
2637 }
2638
2639 *mlx_seg_len =
2640 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
2641 return 0;
2642}
2643
2644static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
2645{
2646 unsigned cur;
2647 struct mlx4_ib_cq *cq;
2648
2649 cur = wq->head - wq->tail;
2650 if (likely(cur + nreq < wq->max_post))
2651 return 0;
2652
2653 cq = to_mcq(ib_cq);
2654 spin_lock(&cq->lock);
2655 cur = wq->head - wq->tail;
2656 spin_unlock(&cq->lock);
2657
2658 return cur + nreq >= wq->max_post;
2659}
2660
2661static __be32 convert_access(int acc)
2662{
2663 return (acc & IB_ACCESS_REMOTE_ATOMIC ?
2664 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC) : 0) |
2665 (acc & IB_ACCESS_REMOTE_WRITE ?
2666 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) |
2667 (acc & IB_ACCESS_REMOTE_READ ?
2668 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ) : 0) |
2669 (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) |
2670 cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
2671}
2672
2673static void set_reg_seg(struct mlx4_wqe_fmr_seg *fseg,
2674 struct ib_reg_wr *wr)
2675{
2676 struct mlx4_ib_mr *mr = to_mmr(wr->mr);
2677
2678 fseg->flags = convert_access(wr->access);
2679 fseg->mem_key = cpu_to_be32(wr->key);
2680 fseg->buf_list = cpu_to_be64(mr->page_map);
2681 fseg->start_addr = cpu_to_be64(mr->ibmr.iova);
2682 fseg->reg_len = cpu_to_be64(mr->ibmr.length);
2683 fseg->offset = 0;
2684 fseg->page_size = cpu_to_be32(ilog2(mr->ibmr.page_size));
2685 fseg->reserved[0] = 0;
2686 fseg->reserved[1] = 0;
2687}
2688
2689static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
2690{
2691 memset(iseg, 0, sizeof(*iseg));
2692 iseg->mem_key = cpu_to_be32(rkey);
2693}
2694
2695static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
2696 u64 remote_addr, u32 rkey)
2697{
2698 rseg->raddr = cpu_to_be64(remote_addr);
2699 rseg->rkey = cpu_to_be32(rkey);
2700 rseg->reserved = 0;
2701}
2702
2703static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg,
2704 struct ib_atomic_wr *wr)
2705{
2706 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
2707 aseg->swap_add = cpu_to_be64(wr->swap);
2708 aseg->compare = cpu_to_be64(wr->compare_add);
2709 } else if (wr->wr.opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
2710 aseg->swap_add = cpu_to_be64(wr->compare_add);
2711 aseg->compare = cpu_to_be64(wr->compare_add_mask);
2712 } else {
2713 aseg->swap_add = cpu_to_be64(wr->compare_add);
2714 aseg->compare = 0;
2715 }
2716
2717}
2718
2719static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
2720 struct ib_atomic_wr *wr)
2721{
2722 aseg->swap_add = cpu_to_be64(wr->swap);
2723 aseg->swap_add_mask = cpu_to_be64(wr->swap_mask);
2724 aseg->compare = cpu_to_be64(wr->compare_add);
2725 aseg->compare_mask = cpu_to_be64(wr->compare_add_mask);
2726}
2727
2728static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
2729 struct ib_ud_wr *wr)
2730{
2731 memcpy(dseg->av, &to_mah(wr->ah)->av, sizeof (struct mlx4_av));
2732 dseg->dqpn = cpu_to_be32(wr->remote_qpn);
2733 dseg->qkey = cpu_to_be32(wr->remote_qkey);
2734 dseg->vlan = to_mah(wr->ah)->av.eth.vlan;
2735 memcpy(dseg->mac, to_mah(wr->ah)->av.eth.mac, 6);
2736}
2737
2738static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev,
2739 struct mlx4_wqe_datagram_seg *dseg,
2740 struct ib_ud_wr *wr,
2741 enum mlx4_ib_qp_type qpt)
2742{
2743 union mlx4_ext_av *av = &to_mah(wr->ah)->av;
2744 struct mlx4_av sqp_av = {0};
2745 int port = *((u8 *) &av->ib.port_pd) & 0x3;
2746
2747
2748 sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000);
2749 sqp_av.g_slid = av->ib.g_slid & 0x7f;
2750 sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel &
2751 cpu_to_be32(0xf0000000);
2752
2753 memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av));
2754 if (qpt == MLX4_IB_QPT_PROXY_GSI)
2755 dseg->dqpn = cpu_to_be32(dev->dev->caps.qp1_tunnel[port - 1]);
2756 else
2757 dseg->dqpn = cpu_to_be32(dev->dev->caps.qp0_tunnel[port - 1]);
2758
2759 dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY);
2760}
2761
2762static void build_tunnel_header(struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len)
2763{
2764 struct mlx4_wqe_inline_seg *inl = wqe;
2765 struct mlx4_ib_tunnel_header hdr;
2766 struct mlx4_ib_ah *ah = to_mah(wr->ah);
2767 int spc;
2768 int i;
2769
2770 memcpy(&hdr.av, &ah->av, sizeof hdr.av);
2771 hdr.remote_qpn = cpu_to_be32(wr->remote_qpn);
2772 hdr.pkey_index = cpu_to_be16(wr->pkey_index);
2773 hdr.qkey = cpu_to_be32(wr->remote_qkey);
2774 memcpy(hdr.mac, ah->av.eth.mac, 6);
2775 hdr.vlan = ah->av.eth.vlan;
2776
2777 spc = MLX4_INLINE_ALIGN -
2778 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2779 if (sizeof (hdr) <= spc) {
2780 memcpy(inl + 1, &hdr, sizeof (hdr));
2781 wmb();
2782 inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr));
2783 i = 1;
2784 } else {
2785 memcpy(inl + 1, &hdr, spc);
2786 wmb();
2787 inl->byte_count = cpu_to_be32(1 << 31 | spc);
2788
2789 inl = (void *) (inl + 1) + spc;
2790 memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc);
2791 wmb();
2792 inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc));
2793 i = 2;
2794 }
2795
2796 *mlx_seg_len =
2797 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16);
2798}
2799
2800static void set_mlx_icrc_seg(void *dseg)
2801{
2802 u32 *t = dseg;
2803 struct mlx4_wqe_inline_seg *iseg = dseg;
2804
2805 t[1] = 0;
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815 wmb();
2816
2817 iseg->byte_count = cpu_to_be32((1 << 31) | 4);
2818}
2819
2820static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
2821{
2822 dseg->lkey = cpu_to_be32(sg->lkey);
2823 dseg->addr = cpu_to_be64(sg->addr);
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833 wmb();
2834
2835 dseg->byte_count = cpu_to_be32(sg->length);
2836}
2837
2838static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
2839{
2840 dseg->byte_count = cpu_to_be32(sg->length);
2841 dseg->lkey = cpu_to_be32(sg->lkey);
2842 dseg->addr = cpu_to_be64(sg->addr);
2843}
2844
2845static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_ud_wr *wr,
2846 struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
2847 __be32 *lso_hdr_sz, __be32 *blh)
2848{
2849 unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16);
2850
2851 if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
2852 *blh = cpu_to_be32(1 << 6);
2853
2854 if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
2855 wr->wr.num_sge > qp->sq.max_gs - (halign >> 4)))
2856 return -EINVAL;
2857
2858 memcpy(wqe->header, wr->header, wr->hlen);
2859
2860 *lso_hdr_sz = cpu_to_be32(wr->mss << 16 | wr->hlen);
2861 *lso_seg_len = halign;
2862 return 0;
2863}
2864
2865static __be32 send_ieth(struct ib_send_wr *wr)
2866{
2867 switch (wr->opcode) {
2868 case IB_WR_SEND_WITH_IMM:
2869 case IB_WR_RDMA_WRITE_WITH_IMM:
2870 return wr->ex.imm_data;
2871
2872 case IB_WR_SEND_WITH_INV:
2873 return cpu_to_be32(wr->ex.invalidate_rkey);
2874
2875 default:
2876 return 0;
2877 }
2878}
2879
2880static void add_zero_len_inline(void *wqe)
2881{
2882 struct mlx4_wqe_inline_seg *inl = wqe;
2883 memset(wqe, 0, 16);
2884 inl->byte_count = cpu_to_be32(1 << 31);
2885}
2886
2887int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2888 struct ib_send_wr **bad_wr)
2889{
2890 struct mlx4_ib_qp *qp = to_mqp(ibqp);
2891 void *wqe;
2892 struct mlx4_wqe_ctrl_seg *ctrl;
2893 struct mlx4_wqe_data_seg *dseg;
2894 unsigned long flags;
2895 int nreq;
2896 int err = 0;
2897 unsigned ind;
2898 int uninitialized_var(stamp);
2899 int uninitialized_var(size);
2900 unsigned uninitialized_var(seglen);
2901 __be32 dummy;
2902 __be32 *lso_wqe;
2903 __be32 uninitialized_var(lso_hdr_sz);
2904 __be32 blh;
2905 int i;
2906 struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
2907
2908 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2909 struct mlx4_ib_sqp *sqp = to_msqp(qp);
2910
2911 if (sqp->roce_v2_gsi) {
2912 struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
2913 struct ib_gid_attr gid_attr;
2914 union ib_gid gid;
2915
2916 if (!ib_get_cached_gid(ibqp->device,
2917 be32_to_cpu(ah->av.ib.port_pd) >> 24,
2918 ah->av.ib.gid_index, &gid,
2919 &gid_attr)) {
2920 if (gid_attr.ndev)
2921 dev_put(gid_attr.ndev);
2922 qp = (gid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
2923 to_mqp(sqp->roce_v2_gsi) : qp;
2924 } else {
2925 pr_err("Failed to get gid at index %d. RoCEv2 will not work properly\n",
2926 ah->av.ib.gid_index);
2927 }
2928 }
2929 }
2930
2931 spin_lock_irqsave(&qp->sq.lock, flags);
2932 if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
2933 err = -EIO;
2934 *bad_wr = wr;
2935 nreq = 0;
2936 goto out;
2937 }
2938
2939 ind = qp->sq_next_wqe;
2940
2941 for (nreq = 0; wr; ++nreq, wr = wr->next) {
2942 lso_wqe = &dummy;
2943 blh = 0;
2944
2945 if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
2946 err = -ENOMEM;
2947 *bad_wr = wr;
2948 goto out;
2949 }
2950
2951 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
2952 err = -EINVAL;
2953 *bad_wr = wr;
2954 goto out;
2955 }
2956
2957 ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
2958 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
2959
2960 ctrl->srcrb_flags =
2961 (wr->send_flags & IB_SEND_SIGNALED ?
2962 cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
2963 (wr->send_flags & IB_SEND_SOLICITED ?
2964 cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
2965 ((wr->send_flags & IB_SEND_IP_CSUM) ?
2966 cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
2967 MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
2968 qp->sq_signal_bits;
2969
2970 ctrl->imm = send_ieth(wr);
2971
2972 wqe += sizeof *ctrl;
2973 size = sizeof *ctrl / 16;
2974
2975 switch (qp->mlx4_ib_qp_type) {
2976 case MLX4_IB_QPT_RC:
2977 case MLX4_IB_QPT_UC:
2978 switch (wr->opcode) {
2979 case IB_WR_ATOMIC_CMP_AND_SWP:
2980 case IB_WR_ATOMIC_FETCH_AND_ADD:
2981 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
2982 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
2983 atomic_wr(wr)->rkey);
2984 wqe += sizeof (struct mlx4_wqe_raddr_seg);
2985
2986 set_atomic_seg(wqe, atomic_wr(wr));
2987 wqe += sizeof (struct mlx4_wqe_atomic_seg);
2988
2989 size += (sizeof (struct mlx4_wqe_raddr_seg) +
2990 sizeof (struct mlx4_wqe_atomic_seg)) / 16;
2991
2992 break;
2993
2994 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
2995 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
2996 atomic_wr(wr)->rkey);
2997 wqe += sizeof (struct mlx4_wqe_raddr_seg);
2998
2999 set_masked_atomic_seg(wqe, atomic_wr(wr));
3000 wqe += sizeof (struct mlx4_wqe_masked_atomic_seg);
3001
3002 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3003 sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
3004
3005 break;
3006
3007 case IB_WR_RDMA_READ:
3008 case IB_WR_RDMA_WRITE:
3009 case IB_WR_RDMA_WRITE_WITH_IMM:
3010 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,
3011 rdma_wr(wr)->rkey);
3012 wqe += sizeof (struct mlx4_wqe_raddr_seg);
3013 size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
3014 break;
3015
3016 case IB_WR_LOCAL_INV:
3017 ctrl->srcrb_flags |=
3018 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3019 set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
3020 wqe += sizeof (struct mlx4_wqe_local_inval_seg);
3021 size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
3022 break;
3023
3024 case IB_WR_REG_MR:
3025 ctrl->srcrb_flags |=
3026 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3027 set_reg_seg(wqe, reg_wr(wr));
3028 wqe += sizeof(struct mlx4_wqe_fmr_seg);
3029 size += sizeof(struct mlx4_wqe_fmr_seg) / 16;
3030 break;
3031
3032 default:
3033
3034 break;
3035 }
3036 break;
3037
3038 case MLX4_IB_QPT_TUN_SMI_OWNER:
3039 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3040 ctrl, &seglen);
3041 if (unlikely(err)) {
3042 *bad_wr = wr;
3043 goto out;
3044 }
3045 wqe += seglen;
3046 size += seglen / 16;
3047 break;
3048 case MLX4_IB_QPT_TUN_SMI:
3049 case MLX4_IB_QPT_TUN_GSI:
3050
3051 set_datagram_seg(wqe, ud_wr(wr));
3052
3053 *(__be32 *) wqe |= cpu_to_be32(0x80000000);
3054 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3055 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3056 break;
3057 case MLX4_IB_QPT_UD:
3058 set_datagram_seg(wqe, ud_wr(wr));
3059 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3060 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3061
3062 if (wr->opcode == IB_WR_LSO) {
3063 err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen,
3064 &lso_hdr_sz, &blh);
3065 if (unlikely(err)) {
3066 *bad_wr = wr;
3067 goto out;
3068 }
3069 lso_wqe = (__be32 *) wqe;
3070 wqe += seglen;
3071 size += seglen / 16;
3072 }
3073 break;
3074
3075 case MLX4_IB_QPT_PROXY_SMI_OWNER:
3076 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3077 ctrl, &seglen);
3078 if (unlikely(err)) {
3079 *bad_wr = wr;
3080 goto out;
3081 }
3082 wqe += seglen;
3083 size += seglen / 16;
3084
3085 add_zero_len_inline(wqe);
3086 wqe += 16;
3087 size++;
3088 build_tunnel_header(ud_wr(wr), wqe, &seglen);
3089 wqe += seglen;
3090 size += seglen / 16;
3091 break;
3092 case MLX4_IB_QPT_PROXY_SMI:
3093 case MLX4_IB_QPT_PROXY_GSI:
3094
3095
3096
3097
3098 set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe,
3099 ud_wr(wr),
3100 qp->mlx4_ib_qp_type);
3101 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3102 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3103 build_tunnel_header(ud_wr(wr), wqe, &seglen);
3104 wqe += seglen;
3105 size += seglen / 16;
3106 break;
3107
3108 case MLX4_IB_QPT_SMI:
3109 case MLX4_IB_QPT_GSI:
3110 err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl,
3111 &seglen);
3112 if (unlikely(err)) {
3113 *bad_wr = wr;
3114 goto out;
3115 }
3116 wqe += seglen;
3117 size += seglen / 16;
3118 break;
3119
3120 default:
3121 break;
3122 }
3123
3124
3125
3126
3127
3128
3129
3130
3131 dseg = wqe;
3132 dseg += wr->num_sge - 1;
3133 size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
3134
3135
3136 if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
3137 qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI ||
3138 qp->mlx4_ib_qp_type &
3139 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) {
3140 set_mlx_icrc_seg(dseg + 1);
3141 size += sizeof (struct mlx4_wqe_data_seg) / 16;
3142 }
3143
3144 for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
3145 set_data_seg(dseg, wr->sg_list + i);
3146
3147
3148
3149
3150
3151
3152 wmb();
3153 *lso_wqe = lso_hdr_sz;
3154
3155 ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ?
3156 MLX4_WQE_CTRL_FENCE : 0) | size;
3157
3158
3159
3160
3161
3162
3163 wmb();
3164
3165 if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
3166 *bad_wr = wr;
3167 err = -EINVAL;
3168 goto out;
3169 }
3170
3171 ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
3172 (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
3173
3174 stamp = ind + qp->sq_spare_wqes;
3175 ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186 if (wr->next) {
3187 stamp_send_wqe(qp, stamp, size * 16);
3188 ind = pad_wraparound(qp, ind);
3189 }
3190 }
3191
3192out:
3193 if (likely(nreq)) {
3194 qp->sq.head += nreq;
3195
3196
3197
3198
3199
3200 wmb();
3201
3202 writel(qp->doorbell_qpn,
3203 to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
3204
3205
3206
3207
3208
3209 mmiowb();
3210
3211 stamp_send_wqe(qp, stamp, size * 16);
3212
3213 ind = pad_wraparound(qp, ind);
3214 qp->sq_next_wqe = ind;
3215 }
3216
3217 spin_unlock_irqrestore(&qp->sq.lock, flags);
3218
3219 return err;
3220}
3221
3222int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3223 struct ib_recv_wr **bad_wr)
3224{
3225 struct mlx4_ib_qp *qp = to_mqp(ibqp);
3226 struct mlx4_wqe_data_seg *scat;
3227 unsigned long flags;
3228 int err = 0;
3229 int nreq;
3230 int ind;
3231 int max_gs;
3232 int i;
3233 struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3234
3235 max_gs = qp->rq.max_gs;
3236 spin_lock_irqsave(&qp->rq.lock, flags);
3237
3238 if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
3239 err = -EIO;
3240 *bad_wr = wr;
3241 nreq = 0;
3242 goto out;
3243 }
3244
3245 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
3246
3247 for (nreq = 0; wr; ++nreq, wr = wr->next) {
3248 if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
3249 err = -ENOMEM;
3250 *bad_wr = wr;
3251 goto out;
3252 }
3253
3254 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
3255 err = -EINVAL;
3256 *bad_wr = wr;
3257 goto out;
3258 }
3259
3260 scat = get_recv_wqe(qp, ind);
3261
3262 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
3263 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
3264 ib_dma_sync_single_for_device(ibqp->device,
3265 qp->sqp_proxy_rcv[ind].map,
3266 sizeof (struct mlx4_ib_proxy_sqp_hdr),
3267 DMA_FROM_DEVICE);
3268 scat->byte_count =
3269 cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr));
3270
3271 scat->lkey = cpu_to_be32(wr->sg_list->lkey);
3272 scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map);
3273 scat++;
3274 max_gs--;
3275 }
3276
3277 for (i = 0; i < wr->num_sge; ++i)
3278 __set_data_seg(scat + i, wr->sg_list + i);
3279
3280 if (i < max_gs) {
3281 scat[i].byte_count = 0;
3282 scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
3283 scat[i].addr = 0;
3284 }
3285
3286 qp->rq.wrid[ind] = wr->wr_id;
3287
3288 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
3289 }
3290
3291out:
3292 if (likely(nreq)) {
3293 qp->rq.head += nreq;
3294
3295
3296
3297
3298
3299 wmb();
3300
3301 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
3302 }
3303
3304 spin_unlock_irqrestore(&qp->rq.lock, flags);
3305
3306 return err;
3307}
3308
3309static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
3310{
3311 switch (mlx4_state) {
3312 case MLX4_QP_STATE_RST: return IB_QPS_RESET;
3313 case MLX4_QP_STATE_INIT: return IB_QPS_INIT;
3314 case MLX4_QP_STATE_RTR: return IB_QPS_RTR;
3315 case MLX4_QP_STATE_RTS: return IB_QPS_RTS;
3316 case MLX4_QP_STATE_SQ_DRAINING:
3317 case MLX4_QP_STATE_SQD: return IB_QPS_SQD;
3318 case MLX4_QP_STATE_SQER: return IB_QPS_SQE;
3319 case MLX4_QP_STATE_ERR: return IB_QPS_ERR;
3320 default: return -1;
3321 }
3322}
3323
3324static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
3325{
3326 switch (mlx4_mig_state) {
3327 case MLX4_QP_PM_ARMED: return IB_MIG_ARMED;
3328 case MLX4_QP_PM_REARM: return IB_MIG_REARM;
3329 case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
3330 default: return -1;
3331 }
3332}
3333
3334static int to_ib_qp_access_flags(int mlx4_flags)
3335{
3336 int ib_flags = 0;
3337
3338 if (mlx4_flags & MLX4_QP_BIT_RRE)
3339 ib_flags |= IB_ACCESS_REMOTE_READ;
3340 if (mlx4_flags & MLX4_QP_BIT_RWE)
3341 ib_flags |= IB_ACCESS_REMOTE_WRITE;
3342 if (mlx4_flags & MLX4_QP_BIT_RAE)
3343 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
3344
3345 return ib_flags;
3346}
3347
3348static void to_ib_ah_attr(struct mlx4_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr,
3349 struct mlx4_qp_path *path)
3350{
3351 struct mlx4_dev *dev = ibdev->dev;
3352 int is_eth;
3353
3354 memset(ib_ah_attr, 0, sizeof *ib_ah_attr);
3355 ib_ah_attr->port_num = path->sched_queue & 0x40 ? 2 : 1;
3356
3357 if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports)
3358 return;
3359
3360 is_eth = rdma_port_get_link_layer(&ibdev->ib_dev, ib_ah_attr->port_num) ==
3361 IB_LINK_LAYER_ETHERNET;
3362 if (is_eth)
3363 ib_ah_attr->sl = ((path->sched_queue >> 3) & 0x7) |
3364 ((path->sched_queue & 4) << 1);
3365 else
3366 ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf;
3367
3368 ib_ah_attr->dlid = be16_to_cpu(path->rlid);
3369 ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f;
3370 ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0;
3371 ib_ah_attr->ah_flags = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
3372 if (ib_ah_attr->ah_flags) {
3373 ib_ah_attr->grh.sgid_index = path->mgid_index;
3374 ib_ah_attr->grh.hop_limit = path->hop_limit;
3375 ib_ah_attr->grh.traffic_class =
3376 (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff;
3377 ib_ah_attr->grh.flow_label =
3378 be32_to_cpu(path->tclass_flowlabel) & 0xfffff;
3379 memcpy(ib_ah_attr->grh.dgid.raw,
3380 path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
3381 }
3382}
3383
3384int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
3385 struct ib_qp_init_attr *qp_init_attr)
3386{
3387 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
3388 struct mlx4_ib_qp *qp = to_mqp(ibqp);
3389 struct mlx4_qp_context context;
3390 int mlx4_state;
3391 int err = 0;
3392
3393 mutex_lock(&qp->mutex);
3394
3395 if (qp->state == IB_QPS_RESET) {
3396 qp_attr->qp_state = IB_QPS_RESET;
3397 goto done;
3398 }
3399
3400 err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
3401 if (err) {
3402 err = -EINVAL;
3403 goto out;
3404 }
3405
3406 mlx4_state = be32_to_cpu(context.flags) >> 28;
3407
3408 qp->state = to_ib_qp_state(mlx4_state);
3409 qp_attr->qp_state = qp->state;
3410 qp_attr->path_mtu = context.mtu_msgmax >> 5;
3411 qp_attr->path_mig_state =
3412 to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
3413 qp_attr->qkey = be32_to_cpu(context.qkey);
3414 qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
3415 qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff;
3416 qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff;
3417 qp_attr->qp_access_flags =
3418 to_ib_qp_access_flags(be32_to_cpu(context.params2));
3419
3420 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
3421 to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
3422 to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
3423 qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
3424 qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num;
3425 }
3426
3427 qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
3428 if (qp_attr->qp_state == IB_QPS_INIT)
3429 qp_attr->port_num = qp->port;
3430 else
3431 qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
3432
3433
3434 qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
3435
3436 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
3437
3438 qp_attr->max_dest_rd_atomic =
3439 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
3440 qp_attr->min_rnr_timer =
3441 (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
3442 qp_attr->timeout = context.pri_path.ackto >> 3;
3443 qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7;
3444 qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7;
3445 qp_attr->alt_timeout = context.alt_path.ackto >> 3;
3446
3447done:
3448 qp_attr->cur_qp_state = qp_attr->qp_state;
3449 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
3450 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
3451
3452 if (!ibqp->uobject) {
3453 qp_attr->cap.max_send_wr = qp->sq.wqe_cnt;
3454 qp_attr->cap.max_send_sge = qp->sq.max_gs;
3455 } else {
3456 qp_attr->cap.max_send_wr = 0;
3457 qp_attr->cap.max_send_sge = 0;
3458 }
3459
3460
3461
3462
3463
3464 qp_attr->cap.max_inline_data = 0;
3465
3466 qp_init_attr->cap = qp_attr->cap;
3467
3468 qp_init_attr->create_flags = 0;
3469 if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
3470 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
3471
3472 if (qp->flags & MLX4_IB_QP_LSO)
3473 qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
3474
3475 if (qp->flags & MLX4_IB_QP_NETIF)
3476 qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP;
3477
3478 qp_init_attr->sq_sig_type =
3479 qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ?
3480 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
3481
3482out:
3483 mutex_unlock(&qp->mutex);
3484 return err;
3485}
3486
3487