1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/log2.h>
35#include <linux/etherdevice.h>
36#include <net/ip.h>
37#include <linux/slab.h>
38#include <linux/netdevice.h>
39
40#include <rdma/ib_cache.h>
41#include <rdma/ib_pack.h>
42#include <rdma/ib_addr.h>
43#include <rdma/ib_mad.h>
44
45#include <linux/mlx4/driver.h>
46#include <linux/mlx4/qp.h>
47
48#include "mlx4_ib.h"
49#include <rdma/mlx4-abi.h>
50
51static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
52 struct mlx4_ib_cq *recv_cq);
53static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
54 struct mlx4_ib_cq *recv_cq);
55static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state);
56
57enum {
58 MLX4_IB_ACK_REQ_FREQ = 8,
59};
60
61enum {
62 MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,
63 MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
64 MLX4_IB_LINK_TYPE_IB = 0,
65 MLX4_IB_LINK_TYPE_ETH = 1
66};
67
68enum {
69
70
71
72
73
74
75 MLX4_IB_UD_HEADER_SIZE = 82,
76 MLX4_IB_LSO_HEADER_SPARE = 128,
77};
78
79struct mlx4_ib_sqp {
80 struct mlx4_ib_qp qp;
81 int pkey_index;
82 u32 qkey;
83 u32 send_psn;
84 struct ib_ud_header ud_header;
85 u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
86 struct ib_qp *roce_v2_gsi;
87};
88
89enum {
90 MLX4_IB_MIN_SQ_STRIDE = 6,
91 MLX4_IB_CACHE_LINE_SIZE = 64,
92};
93
94enum {
95 MLX4_RAW_QP_MTU = 7,
96 MLX4_RAW_QP_MSGMAX = 31,
97};
98
99#ifndef ETH_ALEN
100#define ETH_ALEN 6
101#endif
102
103static const __be32 mlx4_ib_opcode[] = {
104 [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND),
105 [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO),
106 [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM),
107 [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
108 [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
109 [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ),
110 [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
111 [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
112 [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
113 [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
114 [IB_WR_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR),
115 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
116 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
117};
118
119enum mlx4_ib_source_type {
120 MLX4_IB_QP_SRC = 0,
121 MLX4_IB_RWQ_SRC = 1,
122};
123
124static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
125{
126 return container_of(mqp, struct mlx4_ib_sqp, qp);
127}
128
129static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
130{
131 if (!mlx4_is_master(dev->dev))
132 return 0;
133
134 return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn &&
135 qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn +
136 8 * MLX4_MFUNC_MAX;
137}
138
139static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
140{
141 int proxy_sqp = 0;
142 int real_sqp = 0;
143 int i;
144
145 real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
146 qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
147 qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3);
148 if (real_sqp)
149 return 1;
150
151 if (mlx4_is_mfunc(dev->dev)) {
152 for (i = 0; i < dev->dev->caps.num_ports; i++) {
153 if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy ||
154 qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp1_proxy) {
155 proxy_sqp = 1;
156 break;
157 }
158 }
159 }
160 if (proxy_sqp)
161 return 1;
162
163 return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP);
164}
165
166
167static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
168{
169 int proxy_qp0 = 0;
170 int real_qp0 = 0;
171 int i;
172
173 real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
174 qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
175 qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1);
176 if (real_qp0)
177 return 1;
178
179 if (mlx4_is_mfunc(dev->dev)) {
180 for (i = 0; i < dev->dev->caps.num_ports; i++) {
181 if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy) {
182 proxy_qp0 = 1;
183 break;
184 }
185 }
186 }
187 return proxy_qp0;
188}
189
190static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
191{
192 return mlx4_buf_offset(&qp->buf, offset);
193}
194
195static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
196{
197 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
198}
199
200static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
201{
202 return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
203}
204
205
206
207
208
209
210
211
212
213
214static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
215{
216 __be32 *wqe;
217 int i;
218 int s;
219 int ind;
220 void *buf;
221 __be32 stamp;
222 struct mlx4_wqe_ctrl_seg *ctrl;
223
224 if (qp->sq_max_wqes_per_wr > 1) {
225 s = roundup(size, 1U << qp->sq.wqe_shift);
226 for (i = 0; i < s; i += 64) {
227 ind = (i >> qp->sq.wqe_shift) + n;
228 stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
229 cpu_to_be32(0xffffffff);
230 buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
231 wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
232 *wqe = stamp;
233 }
234 } else {
235 ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
236 s = (ctrl->qpn_vlan.fence_size & 0x3f) << 4;
237 for (i = 64; i < s; i += 64) {
238 wqe = buf + i;
239 *wqe = cpu_to_be32(0xffffffff);
240 }
241 }
242}
243
244static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
245{
246 struct mlx4_wqe_ctrl_seg *ctrl;
247 struct mlx4_wqe_inline_seg *inl;
248 void *wqe;
249 int s;
250
251 ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
252 s = sizeof(struct mlx4_wqe_ctrl_seg);
253
254 if (qp->ibqp.qp_type == IB_QPT_UD) {
255 struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
256 struct mlx4_av *av = (struct mlx4_av *)dgram->av;
257 memset(dgram, 0, sizeof *dgram);
258 av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
259 s += sizeof(struct mlx4_wqe_datagram_seg);
260 }
261
262
263 if (size > s) {
264 inl = wqe + s;
265 inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
266 }
267 ctrl->srcrb_flags = 0;
268 ctrl->qpn_vlan.fence_size = size / 16;
269
270
271
272
273 wmb();
274
275 ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
276 (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
277
278 stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
279}
280
281
282static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
283{
284 unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
285 if (unlikely(s < qp->sq_max_wqes_per_wr)) {
286 post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
287 ind += s;
288 }
289 return ind;
290}
291
292static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
293{
294 struct ib_event event;
295 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
296
297 if (type == MLX4_EVENT_TYPE_PATH_MIG)
298 to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
299
300 if (ibqp->event_handler) {
301 event.device = ibqp->device;
302 event.element.qp = ibqp;
303 switch (type) {
304 case MLX4_EVENT_TYPE_PATH_MIG:
305 event.event = IB_EVENT_PATH_MIG;
306 break;
307 case MLX4_EVENT_TYPE_COMM_EST:
308 event.event = IB_EVENT_COMM_EST;
309 break;
310 case MLX4_EVENT_TYPE_SQ_DRAINED:
311 event.event = IB_EVENT_SQ_DRAINED;
312 break;
313 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
314 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
315 break;
316 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
317 event.event = IB_EVENT_QP_FATAL;
318 break;
319 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
320 event.event = IB_EVENT_PATH_MIG_ERR;
321 break;
322 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
323 event.event = IB_EVENT_QP_REQ_ERR;
324 break;
325 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
326 event.event = IB_EVENT_QP_ACCESS_ERR;
327 break;
328 default:
329 pr_warn("Unexpected event type %d "
330 "on QP %06x\n", type, qp->qpn);
331 return;
332 }
333
334 ibqp->event_handler(&event, ibqp->qp_context);
335 }
336}
337
338static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
339{
340 pr_warn_ratelimited("Unexpected event type %d on WQ 0x%06x. Events are not supported for WQs\n",
341 type, qp->qpn);
342}
343
344static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
345{
346
347
348
349
350
351
352 switch (type) {
353 case MLX4_IB_QPT_UD:
354 return sizeof (struct mlx4_wqe_ctrl_seg) +
355 sizeof (struct mlx4_wqe_datagram_seg) +
356 ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
357 case MLX4_IB_QPT_PROXY_SMI_OWNER:
358 case MLX4_IB_QPT_PROXY_SMI:
359 case MLX4_IB_QPT_PROXY_GSI:
360 return sizeof (struct mlx4_wqe_ctrl_seg) +
361 sizeof (struct mlx4_wqe_datagram_seg) + 64;
362 case MLX4_IB_QPT_TUN_SMI_OWNER:
363 case MLX4_IB_QPT_TUN_GSI:
364 return sizeof (struct mlx4_wqe_ctrl_seg) +
365 sizeof (struct mlx4_wqe_datagram_seg);
366
367 case MLX4_IB_QPT_UC:
368 return sizeof (struct mlx4_wqe_ctrl_seg) +
369 sizeof (struct mlx4_wqe_raddr_seg);
370 case MLX4_IB_QPT_RC:
371 return sizeof (struct mlx4_wqe_ctrl_seg) +
372 sizeof (struct mlx4_wqe_masked_atomic_seg) +
373 sizeof (struct mlx4_wqe_raddr_seg);
374 case MLX4_IB_QPT_SMI:
375 case MLX4_IB_QPT_GSI:
376 return sizeof (struct mlx4_wqe_ctrl_seg) +
377 ALIGN(MLX4_IB_UD_HEADER_SIZE +
378 DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
379 MLX4_INLINE_ALIGN) *
380 sizeof (struct mlx4_wqe_inline_seg),
381 sizeof (struct mlx4_wqe_data_seg)) +
382 ALIGN(4 +
383 sizeof (struct mlx4_wqe_inline_seg),
384 sizeof (struct mlx4_wqe_data_seg));
385 default:
386 return sizeof (struct mlx4_wqe_ctrl_seg);
387 }
388}
389
390static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
391 int is_user, int has_rq, struct mlx4_ib_qp *qp,
392 u32 inl_recv_sz)
393{
394
395 if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
396 cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
397 return -EINVAL;
398
399 if (!has_rq) {
400 if (cap->max_recv_wr || inl_recv_sz)
401 return -EINVAL;
402
403 qp->rq.wqe_cnt = qp->rq.max_gs = 0;
404 } else {
405 u32 max_inl_recv_sz = dev->dev->caps.max_rq_sg *
406 sizeof(struct mlx4_wqe_data_seg);
407 u32 wqe_size;
408
409
410 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge ||
411 inl_recv_sz > max_inl_recv_sz))
412 return -EINVAL;
413
414 qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));
415 qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));
416 wqe_size = qp->rq.max_gs * sizeof(struct mlx4_wqe_data_seg);
417 qp->rq.wqe_shift = ilog2(max_t(u32, wqe_size, inl_recv_sz));
418 }
419
420
421 if (is_user) {
422 cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
423 cap->max_recv_sge = qp->rq.max_gs;
424 } else {
425 cap->max_recv_wr = qp->rq.max_post =
426 min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
427 cap->max_recv_sge = min(qp->rq.max_gs,
428 min(dev->dev->caps.max_sq_sg,
429 dev->dev->caps.max_rq_sg));
430 }
431
432 return 0;
433}
434
435static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
436 enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp,
437 bool shrink_wqe)
438{
439 int s;
440
441
442 if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) ||
443 cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
444 cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
445 sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
446 return -EINVAL;
447
448
449
450
451
452 if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI ||
453 type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) &&
454 cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
455 return -EINVAL;
456
457 s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
458 cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
459 send_wqe_overhead(type, qp->flags);
460
461 if (s > dev->dev->caps.max_sq_desc_sz)
462 return -EINVAL;
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495 if (shrink_wqe && dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
496 qp->sq_signal_bits && BITS_PER_LONG == 64 &&
497 type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI &&
498 !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI |
499 MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER)))
500 qp->sq.wqe_shift = ilog2(64);
501 else
502 qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
503
504 for (;;) {
505 qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
506
507
508
509
510
511 qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
512 qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
513 qp->sq_max_wqes_per_wr +
514 qp->sq_spare_wqes);
515
516 if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
517 break;
518
519 if (qp->sq_max_wqes_per_wr <= 1)
520 return -EINVAL;
521
522 ++qp->sq.wqe_shift;
523 }
524
525 qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
526 (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
527 send_wqe_overhead(type, qp->flags)) /
528 sizeof (struct mlx4_wqe_data_seg);
529
530 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
531 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
532 if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
533 qp->rq.offset = 0;
534 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
535 } else {
536 qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
537 qp->sq.offset = 0;
538 }
539
540 cap->max_send_wr = qp->sq.max_post =
541 (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
542 cap->max_send_sge = min(qp->sq.max_gs,
543 min(dev->dev->caps.max_sq_sg,
544 dev->dev->caps.max_rq_sg));
545
546 cap->max_inline_data = 0;
547
548 return 0;
549}
550
551static int set_user_sq_size(struct mlx4_ib_dev *dev,
552 struct mlx4_ib_qp *qp,
553 struct mlx4_ib_create_qp *ucmd)
554{
555
556 if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||
557 ucmd->log_sq_stride >
558 ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
559 ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
560 return -EINVAL;
561
562 qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
563 qp->sq.wqe_shift = ucmd->log_sq_stride;
564
565 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
566 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
567
568 return 0;
569}
570
571static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
572{
573 int i;
574
575 qp->sqp_proxy_rcv =
576 kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt,
577 GFP_KERNEL);
578 if (!qp->sqp_proxy_rcv)
579 return -ENOMEM;
580 for (i = 0; i < qp->rq.wqe_cnt; i++) {
581 qp->sqp_proxy_rcv[i].addr =
582 kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr),
583 GFP_KERNEL);
584 if (!qp->sqp_proxy_rcv[i].addr)
585 goto err;
586 qp->sqp_proxy_rcv[i].map =
587 ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr,
588 sizeof (struct mlx4_ib_proxy_sqp_hdr),
589 DMA_FROM_DEVICE);
590 if (ib_dma_mapping_error(dev, qp->sqp_proxy_rcv[i].map)) {
591 kfree(qp->sqp_proxy_rcv[i].addr);
592 goto err;
593 }
594 }
595 return 0;
596
597err:
598 while (i > 0) {
599 --i;
600 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
601 sizeof (struct mlx4_ib_proxy_sqp_hdr),
602 DMA_FROM_DEVICE);
603 kfree(qp->sqp_proxy_rcv[i].addr);
604 }
605 kfree(qp->sqp_proxy_rcv);
606 qp->sqp_proxy_rcv = NULL;
607 return -ENOMEM;
608}
609
610static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
611{
612 int i;
613
614 for (i = 0; i < qp->rq.wqe_cnt; i++) {
615 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
616 sizeof (struct mlx4_ib_proxy_sqp_hdr),
617 DMA_FROM_DEVICE);
618 kfree(qp->sqp_proxy_rcv[i].addr);
619 }
620 kfree(qp->sqp_proxy_rcv);
621}
622
623static int qp_has_rq(struct ib_qp_init_attr *attr)
624{
625 if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT)
626 return 0;
627
628 return !attr->srq;
629}
630
631static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
632{
633 int i;
634 for (i = 0; i < dev->caps.num_ports; i++) {
635 if (qpn == dev->caps.spec_qps[i].qp0_proxy)
636 return !!dev->caps.spec_qps[i].qp0_qkey;
637 }
638 return 0;
639}
640
641static void mlx4_ib_free_qp_counter(struct mlx4_ib_dev *dev,
642 struct mlx4_ib_qp *qp)
643{
644 mutex_lock(&dev->counters_table[qp->port - 1].mutex);
645 mlx4_counter_free(dev->dev, qp->counter_index->index);
646 list_del(&qp->counter_index->list);
647 mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
648
649 kfree(qp->counter_index);
650 qp->counter_index = NULL;
651}
652
653static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,
654 struct ib_qp_init_attr *init_attr,
655 struct mlx4_ib_create_qp_rss *ucmd)
656{
657 rss_ctx->base_qpn_tbl_sz = init_attr->rwq_ind_tbl->ind_tbl[0]->wq_num |
658 (init_attr->rwq_ind_tbl->log_ind_tbl_size << 24);
659
660 if ((ucmd->rx_hash_function == MLX4_IB_RX_HASH_FUNC_TOEPLITZ) &&
661 (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP)) {
662 memcpy(rss_ctx->rss_key, ucmd->rx_hash_key,
663 MLX4_EN_RSS_KEY_SIZE);
664 } else {
665 pr_debug("RX Hash function is not supported\n");
666 return (-EOPNOTSUPP);
667 }
668
669 if (ucmd->rx_hash_fields_mask & ~(MLX4_IB_RX_HASH_SRC_IPV4 |
670 MLX4_IB_RX_HASH_DST_IPV4 |
671 MLX4_IB_RX_HASH_SRC_IPV6 |
672 MLX4_IB_RX_HASH_DST_IPV6 |
673 MLX4_IB_RX_HASH_SRC_PORT_TCP |
674 MLX4_IB_RX_HASH_DST_PORT_TCP |
675 MLX4_IB_RX_HASH_SRC_PORT_UDP |
676 MLX4_IB_RX_HASH_DST_PORT_UDP |
677 MLX4_IB_RX_HASH_INNER)) {
678 pr_debug("RX Hash fields_mask has unsupported mask (0x%llx)\n",
679 ucmd->rx_hash_fields_mask);
680 return (-EOPNOTSUPP);
681 }
682
683 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) &&
684 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
685 rss_ctx->flags = MLX4_RSS_IPV4;
686 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) ||
687 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
688 pr_debug("RX Hash fields_mask is not supported - both IPv4 SRC and DST must be set\n");
689 return (-EOPNOTSUPP);
690 }
691
692 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) &&
693 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
694 rss_ctx->flags |= MLX4_RSS_IPV6;
695 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) ||
696 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
697 pr_debug("RX Hash fields_mask is not supported - both IPv6 SRC and DST must be set\n");
698 return (-EOPNOTSUPP);
699 }
700
701 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) &&
702 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
703 if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS)) {
704 pr_debug("RX Hash fields_mask for UDP is not supported\n");
705 return (-EOPNOTSUPP);
706 }
707
708 if (rss_ctx->flags & MLX4_RSS_IPV4)
709 rss_ctx->flags |= MLX4_RSS_UDP_IPV4;
710 if (rss_ctx->flags & MLX4_RSS_IPV6)
711 rss_ctx->flags |= MLX4_RSS_UDP_IPV6;
712 if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
713 pr_debug("RX Hash fields_mask is not supported - UDP must be set with IPv4 or IPv6\n");
714 return (-EOPNOTSUPP);
715 }
716 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) ||
717 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
718 pr_debug("RX Hash fields_mask is not supported - both UDP SRC and DST must be set\n");
719 return (-EOPNOTSUPP);
720 }
721
722 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) &&
723 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
724 if (rss_ctx->flags & MLX4_RSS_IPV4)
725 rss_ctx->flags |= MLX4_RSS_TCP_IPV4;
726 if (rss_ctx->flags & MLX4_RSS_IPV6)
727 rss_ctx->flags |= MLX4_RSS_TCP_IPV6;
728 if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
729 pr_debug("RX Hash fields_mask is not supported - TCP must be set with IPv4 or IPv6\n");
730 return (-EOPNOTSUPP);
731 }
732 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) ||
733 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
734 pr_debug("RX Hash fields_mask is not supported - both TCP SRC and DST must be set\n");
735 return (-EOPNOTSUPP);
736 }
737
738 if (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_INNER) {
739 if (dev->dev->caps.tunnel_offload_mode ==
740 MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
741
742
743
744
745 rss_ctx->flags |= MLX4_RSS_BY_INNER_HEADERS_IPONLY;
746 } else {
747 pr_debug("RSS Hash for inner headers isn't supported\n");
748 return (-EOPNOTSUPP);
749 }
750 }
751
752 return 0;
753}
754
755static int create_qp_rss(struct mlx4_ib_dev *dev,
756 struct ib_qp_init_attr *init_attr,
757 struct mlx4_ib_create_qp_rss *ucmd,
758 struct mlx4_ib_qp *qp)
759{
760 int qpn;
761 int err;
762
763 qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
764
765 err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn, 0, qp->mqp.usage);
766 if (err)
767 return err;
768
769 err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
770 if (err)
771 goto err_qpn;
772
773 mutex_init(&qp->mutex);
774
775 INIT_LIST_HEAD(&qp->gid_list);
776 INIT_LIST_HEAD(&qp->steering_rules);
777
778 qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
779 qp->state = IB_QPS_RESET;
780
781
782 qp->sq_no_prefetch = 1;
783 qp->sq.wqe_cnt = 1;
784 qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
785 qp->buf_size = qp->sq.wqe_cnt << MLX4_IB_MIN_SQ_STRIDE;
786 qp->mtt = (to_mqp(
787 (struct ib_qp *)init_attr->rwq_ind_tbl->ind_tbl[0]))->mtt;
788
789 qp->rss_ctx = kzalloc(sizeof(*qp->rss_ctx), GFP_KERNEL);
790 if (!qp->rss_ctx) {
791 err = -ENOMEM;
792 goto err_qp_alloc;
793 }
794
795 err = set_qp_rss(dev, qp->rss_ctx, init_attr, ucmd);
796 if (err)
797 goto err;
798
799 return 0;
800
801err:
802 kfree(qp->rss_ctx);
803
804err_qp_alloc:
805 mlx4_qp_remove(dev->dev, &qp->mqp);
806 mlx4_qp_free(dev->dev, &qp->mqp);
807
808err_qpn:
809 mlx4_qp_release_range(dev->dev, qpn, 1);
810 return err;
811}
812
813static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd,
814 struct ib_qp_init_attr *init_attr,
815 struct ib_udata *udata)
816{
817 struct mlx4_ib_qp *qp;
818 struct mlx4_ib_create_qp_rss ucmd = {};
819 size_t required_cmd_sz;
820 int err;
821
822 if (!udata) {
823 pr_debug("RSS QP with NULL udata\n");
824 return ERR_PTR(-EINVAL);
825 }
826
827 if (udata->outlen)
828 return ERR_PTR(-EOPNOTSUPP);
829
830 required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
831 sizeof(ucmd.reserved1);
832 if (udata->inlen < required_cmd_sz) {
833 pr_debug("invalid inlen\n");
834 return ERR_PTR(-EINVAL);
835 }
836
837 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
838 pr_debug("copy failed\n");
839 return ERR_PTR(-EFAULT);
840 }
841
842 if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
843 return ERR_PTR(-EOPNOTSUPP);
844
845 if (ucmd.comp_mask || ucmd.reserved1)
846 return ERR_PTR(-EOPNOTSUPP);
847
848 if (udata->inlen > sizeof(ucmd) &&
849 !ib_is_udata_cleared(udata, sizeof(ucmd),
850 udata->inlen - sizeof(ucmd))) {
851 pr_debug("inlen is not supported\n");
852 return ERR_PTR(-EOPNOTSUPP);
853 }
854
855 if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
856 pr_debug("RSS QP with unsupported QP type %d\n",
857 init_attr->qp_type);
858 return ERR_PTR(-EOPNOTSUPP);
859 }
860
861 if (init_attr->create_flags) {
862 pr_debug("RSS QP doesn't support create flags\n");
863 return ERR_PTR(-EOPNOTSUPP);
864 }
865
866 if (init_attr->send_cq || init_attr->cap.max_send_wr) {
867 pr_debug("RSS QP with unsupported send attributes\n");
868 return ERR_PTR(-EOPNOTSUPP);
869 }
870
871 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
872 if (!qp)
873 return ERR_PTR(-ENOMEM);
874
875 qp->pri.vid = 0xFFFF;
876 qp->alt.vid = 0xFFFF;
877
878 err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp);
879 if (err) {
880 kfree(qp);
881 return ERR_PTR(err);
882 }
883
884 qp->ibqp.qp_num = qp->mqp.qpn;
885
886 return &qp->ibqp;
887}
888
889
890
891
892
893
894static int mlx4_ib_alloc_wqn(struct mlx4_ib_ucontext *context,
895 struct mlx4_ib_qp *qp, int range_size, int *wqn)
896{
897 struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
898 struct mlx4_wqn_range *range;
899 int err = 0;
900
901 mutex_lock(&context->wqn_ranges_mutex);
902
903 range = list_first_entry_or_null(&context->wqn_ranges_list,
904 struct mlx4_wqn_range, list);
905
906 if (!range || (range->refcount == range->size) || range->dirty) {
907 range = kzalloc(sizeof(*range), GFP_KERNEL);
908 if (!range) {
909 err = -ENOMEM;
910 goto out;
911 }
912
913 err = mlx4_qp_reserve_range(dev->dev, range_size,
914 range_size, &range->base_wqn, 0,
915 qp->mqp.usage);
916 if (err) {
917 kfree(range);
918 goto out;
919 }
920
921 range->size = range_size;
922 list_add(&range->list, &context->wqn_ranges_list);
923 } else if (range_size != 1) {
924
925
926
927
928 err = -EINVAL;
929 goto out;
930 }
931
932 qp->wqn_range = range;
933
934 *wqn = range->base_wqn + range->refcount;
935
936 range->refcount++;
937
938out:
939 mutex_unlock(&context->wqn_ranges_mutex);
940
941 return err;
942}
943
944static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context,
945 struct mlx4_ib_qp *qp, bool dirty_release)
946{
947 struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
948 struct mlx4_wqn_range *range;
949
950 mutex_lock(&context->wqn_ranges_mutex);
951
952 range = qp->wqn_range;
953
954 range->refcount--;
955 if (!range->refcount) {
956 mlx4_qp_release_range(dev->dev, range->base_wqn,
957 range->size);
958 list_del(&range->list);
959 kfree(range);
960 } else if (dirty_release) {
961
962
963
964
965
966 range->dirty = 1;
967 }
968
969 mutex_unlock(&context->wqn_ranges_mutex);
970}
971
972static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
973 enum mlx4_ib_source_type src,
974 struct ib_qp_init_attr *init_attr,
975 struct ib_udata *udata, int sqpn,
976 struct mlx4_ib_qp **caller_qp)
977{
978 int qpn;
979 int err;
980 struct ib_qp_cap backup_cap;
981 struct mlx4_ib_sqp *sqp = NULL;
982 struct mlx4_ib_qp *qp;
983 enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
984 struct mlx4_ib_cq *mcq;
985 unsigned long flags;
986 int range_size = 0;
987
988
989 if (sqpn) {
990 if (mlx4_is_mfunc(dev->dev) &&
991 (!mlx4_is_master(dev->dev) ||
992 !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {
993 if (init_attr->qp_type == IB_QPT_GSI)
994 qp_type = MLX4_IB_QPT_PROXY_GSI;
995 else {
996 if (mlx4_is_master(dev->dev) ||
997 qp0_enabled_vf(dev->dev, sqpn))
998 qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;
999 else
1000 qp_type = MLX4_IB_QPT_PROXY_SMI;
1001 }
1002 }
1003 qpn = sqpn;
1004
1005 init_attr->cap.max_recv_sge++;
1006 } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) {
1007 struct mlx4_ib_qp_tunnel_init_attr *tnl_init =
1008 container_of(init_attr,
1009 struct mlx4_ib_qp_tunnel_init_attr, init_attr);
1010 if ((tnl_init->proxy_qp_type != IB_QPT_SMI &&
1011 tnl_init->proxy_qp_type != IB_QPT_GSI) ||
1012 !mlx4_is_master(dev->dev))
1013 return -EINVAL;
1014 if (tnl_init->proxy_qp_type == IB_QPT_GSI)
1015 qp_type = MLX4_IB_QPT_TUN_GSI;
1016 else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||
1017 mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,
1018 tnl_init->port))
1019 qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;
1020 else
1021 qp_type = MLX4_IB_QPT_TUN_SMI;
1022
1023
1024 qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave
1025 + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1;
1026 sqpn = qpn;
1027 }
1028
1029 if (!*caller_qp) {
1030 if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI ||
1031 (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |
1032 MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {
1033 sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
1034 if (!sqp)
1035 return -ENOMEM;
1036 qp = &sqp->qp;
1037 qp->pri.vid = 0xFFFF;
1038 qp->alt.vid = 0xFFFF;
1039 } else {
1040 qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL);
1041 if (!qp)
1042 return -ENOMEM;
1043 qp->pri.vid = 0xFFFF;
1044 qp->alt.vid = 0xFFFF;
1045 }
1046 } else
1047 qp = *caller_qp;
1048
1049 qp->mlx4_ib_qp_type = qp_type;
1050
1051 mutex_init(&qp->mutex);
1052 spin_lock_init(&qp->sq.lock);
1053 spin_lock_init(&qp->rq.lock);
1054 INIT_LIST_HEAD(&qp->gid_list);
1055 INIT_LIST_HEAD(&qp->steering_rules);
1056
1057 qp->state = IB_QPS_RESET;
1058 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1059 qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
1060
1061
1062 if (pd->uobject) {
1063 union {
1064 struct mlx4_ib_create_qp qp;
1065 struct mlx4_ib_create_wq wq;
1066 } ucmd;
1067 size_t copy_len;
1068 int shift;
1069 int n;
1070
1071 copy_len = (src == MLX4_IB_QP_SRC) ?
1072 sizeof(struct mlx4_ib_create_qp) :
1073 min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
1074
1075 if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
1076 err = -EFAULT;
1077 goto err;
1078 }
1079
1080 if (src == MLX4_IB_RWQ_SRC) {
1081 if (ucmd.wq.comp_mask || ucmd.wq.reserved[0] ||
1082 ucmd.wq.reserved[1] || ucmd.wq.reserved[2]) {
1083 pr_debug("user command isn't supported\n");
1084 err = -EOPNOTSUPP;
1085 goto err;
1086 }
1087
1088 if (ucmd.wq.log_range_size >
1089 ilog2(dev->dev->caps.max_rss_tbl_sz)) {
1090 pr_debug("WQN range size must be equal or smaller than %d\n",
1091 dev->dev->caps.max_rss_tbl_sz);
1092 err = -EOPNOTSUPP;
1093 goto err;
1094 }
1095 range_size = 1 << ucmd.wq.log_range_size;
1096 } else {
1097 qp->inl_recv_sz = ucmd.qp.inl_recv_sz;
1098 }
1099
1100 if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
1101 if (!(dev->dev->caps.flags &
1102 MLX4_DEV_CAP_FLAG_FCS_KEEP)) {
1103 pr_debug("scatter FCS is unsupported\n");
1104 err = -EOPNOTSUPP;
1105 goto err;
1106 }
1107
1108 qp->flags |= MLX4_IB_QP_SCATTER_FCS;
1109 }
1110
1111 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
1112 qp_has_rq(init_attr), qp, qp->inl_recv_sz);
1113 if (err)
1114 goto err;
1115
1116 if (src == MLX4_IB_QP_SRC) {
1117 qp->sq_no_prefetch = ucmd.qp.sq_no_prefetch;
1118
1119 err = set_user_sq_size(dev, qp,
1120 (struct mlx4_ib_create_qp *)
1121 &ucmd);
1122 if (err)
1123 goto err;
1124 } else {
1125 qp->sq_no_prefetch = 1;
1126 qp->sq.wqe_cnt = 1;
1127 qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
1128
1129
1130
1131 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
1132 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
1133 }
1134
1135 qp->umem = ib_umem_get(pd->uobject->context,
1136 (src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :
1137 ucmd.wq.buf_addr, qp->buf_size, 0, 0);
1138 if (IS_ERR(qp->umem)) {
1139 err = PTR_ERR(qp->umem);
1140 goto err;
1141 }
1142
1143 n = ib_umem_page_count(qp->umem);
1144 shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
1145 err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
1146
1147 if (err)
1148 goto err_buf;
1149
1150 err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
1151 if (err)
1152 goto err_mtt;
1153
1154 if (qp_has_rq(init_attr)) {
1155 err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
1156 (src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
1157 ucmd.wq.db_addr, &qp->db);
1158 if (err)
1159 goto err_mtt;
1160 }
1161 qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
1162 } else {
1163 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
1164 qp_has_rq(init_attr), qp, 0);
1165 if (err)
1166 goto err;
1167
1168 qp->sq_no_prefetch = 0;
1169
1170 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
1171 qp->flags |= MLX4_IB_QP_LSO;
1172
1173 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1174 if (dev->steering_support ==
1175 MLX4_STEERING_MODE_DEVICE_MANAGED)
1176 qp->flags |= MLX4_IB_QP_NETIF;
1177 else
1178 goto err;
1179 }
1180
1181 memcpy(&backup_cap, &init_attr->cap, sizeof(backup_cap));
1182 err = set_kernel_sq_size(dev, &init_attr->cap,
1183 qp_type, qp, true);
1184 if (err)
1185 goto err;
1186
1187 if (qp_has_rq(init_attr)) {
1188 err = mlx4_db_alloc(dev->dev, &qp->db, 0);
1189 if (err)
1190 goto err;
1191
1192 *qp->db.db = 0;
1193 }
1194
1195 if (mlx4_buf_alloc(dev->dev, qp->buf_size, qp->buf_size,
1196 &qp->buf)) {
1197 memcpy(&init_attr->cap, &backup_cap,
1198 sizeof(backup_cap));
1199 err = set_kernel_sq_size(dev, &init_attr->cap, qp_type,
1200 qp, false);
1201 if (err)
1202 goto err_db;
1203
1204 if (mlx4_buf_alloc(dev->dev, qp->buf_size,
1205 PAGE_SIZE * 2, &qp->buf)) {
1206 err = -ENOMEM;
1207 goto err_db;
1208 }
1209 }
1210
1211 err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
1212 &qp->mtt);
1213 if (err)
1214 goto err_buf;
1215
1216 err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
1217 if (err)
1218 goto err_mtt;
1219
1220 qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1221 sizeof(u64), GFP_KERNEL);
1222 qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1223 sizeof(u64), GFP_KERNEL);
1224 if (!qp->sq.wrid || !qp->rq.wrid) {
1225 err = -ENOMEM;
1226 goto err_wrid;
1227 }
1228 qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
1229 }
1230
1231 if (sqpn) {
1232 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1233 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
1234 if (alloc_proxy_bufs(pd->device, qp)) {
1235 err = -ENOMEM;
1236 goto err_wrid;
1237 }
1238 }
1239 } else if (src == MLX4_IB_RWQ_SRC) {
1240 err = mlx4_ib_alloc_wqn(to_mucontext(pd->uobject->context), qp,
1241 range_size, &qpn);
1242 if (err)
1243 goto err_wrid;
1244 } else {
1245
1246
1247
1248 if (init_attr->qp_type == IB_QPT_RAW_PACKET)
1249 err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
1250 (init_attr->cap.max_send_wr ?
1251 MLX4_RESERVE_ETH_BF_QP : 0) |
1252 (init_attr->cap.max_recv_wr ?
1253 MLX4_RESERVE_A0_QP : 0),
1254 qp->mqp.usage);
1255 else
1256 if (qp->flags & MLX4_IB_QP_NETIF)
1257 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
1258 else
1259 err = mlx4_qp_reserve_range(dev->dev, 1, 1,
1260 &qpn, 0, qp->mqp.usage);
1261 if (err)
1262 goto err_proxy;
1263 }
1264
1265 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
1266 qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1267
1268 err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
1269 if (err)
1270 goto err_qpn;
1271
1272 if (init_attr->qp_type == IB_QPT_XRC_TGT)
1273 qp->mqp.qpn |= (1 << 23);
1274
1275
1276
1277
1278
1279
1280 qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
1281
1282 qp->mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :
1283 mlx4_ib_wq_event;
1284
1285 if (!*caller_qp)
1286 *caller_qp = qp;
1287
1288 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1289 mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
1290 to_mcq(init_attr->recv_cq));
1291
1292
1293
1294 list_add_tail(&qp->qps_list, &dev->qp_list);
1295
1296
1297
1298 mcq = to_mcq(init_attr->send_cq);
1299 list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
1300 mcq = to_mcq(init_attr->recv_cq);
1301 list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
1302 mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
1303 to_mcq(init_attr->recv_cq));
1304 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1305 return 0;
1306
1307err_qpn:
1308 if (!sqpn) {
1309 if (qp->flags & MLX4_IB_QP_NETIF)
1310 mlx4_ib_steer_qp_free(dev, qpn, 1);
1311 else if (src == MLX4_IB_RWQ_SRC)
1312 mlx4_ib_release_wqn(to_mucontext(pd->uobject->context),
1313 qp, 0);
1314 else
1315 mlx4_qp_release_range(dev->dev, qpn, 1);
1316 }
1317err_proxy:
1318 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1319 free_proxy_bufs(pd->device, qp);
1320err_wrid:
1321 if (pd->uobject) {
1322 if (qp_has_rq(init_attr))
1323 mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
1324 } else {
1325 kvfree(qp->sq.wrid);
1326 kvfree(qp->rq.wrid);
1327 }
1328
1329err_mtt:
1330 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1331
1332err_buf:
1333 if (pd->uobject)
1334 ib_umem_release(qp->umem);
1335 else
1336 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1337
1338err_db:
1339 if (!pd->uobject && qp_has_rq(init_attr))
1340 mlx4_db_free(dev->dev, &qp->db);
1341
1342err:
1343 if (sqp)
1344 kfree(sqp);
1345 else if (!*caller_qp)
1346 kfree(qp);
1347 return err;
1348}
1349
1350static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
1351{
1352 switch (state) {
1353 case IB_QPS_RESET: return MLX4_QP_STATE_RST;
1354 case IB_QPS_INIT: return MLX4_QP_STATE_INIT;
1355 case IB_QPS_RTR: return MLX4_QP_STATE_RTR;
1356 case IB_QPS_RTS: return MLX4_QP_STATE_RTS;
1357 case IB_QPS_SQD: return MLX4_QP_STATE_SQD;
1358 case IB_QPS_SQE: return MLX4_QP_STATE_SQER;
1359 case IB_QPS_ERR: return MLX4_QP_STATE_ERR;
1360 default: return -1;
1361 }
1362}
1363
1364static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1365 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1366{
1367 if (send_cq == recv_cq) {
1368 spin_lock(&send_cq->lock);
1369 __acquire(&recv_cq->lock);
1370 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1371 spin_lock(&send_cq->lock);
1372 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1373 } else {
1374 spin_lock(&recv_cq->lock);
1375 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1376 }
1377}
1378
1379static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1380 __releases(&send_cq->lock) __releases(&recv_cq->lock)
1381{
1382 if (send_cq == recv_cq) {
1383 __release(&recv_cq->lock);
1384 spin_unlock(&send_cq->lock);
1385 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1386 spin_unlock(&recv_cq->lock);
1387 spin_unlock(&send_cq->lock);
1388 } else {
1389 spin_unlock(&send_cq->lock);
1390 spin_unlock(&recv_cq->lock);
1391 }
1392}
1393
1394static void del_gid_entries(struct mlx4_ib_qp *qp)
1395{
1396 struct mlx4_ib_gid_entry *ge, *tmp;
1397
1398 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1399 list_del(&ge->list);
1400 kfree(ge);
1401 }
1402}
1403
1404static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
1405{
1406 if (qp->ibqp.qp_type == IB_QPT_XRC_TGT)
1407 return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd);
1408 else
1409 return to_mpd(qp->ibqp.pd);
1410}
1411
1412static void get_cqs(struct mlx4_ib_qp *qp, enum mlx4_ib_source_type src,
1413 struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
1414{
1415 switch (qp->ibqp.qp_type) {
1416 case IB_QPT_XRC_TGT:
1417 *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq);
1418 *recv_cq = *send_cq;
1419 break;
1420 case IB_QPT_XRC_INI:
1421 *send_cq = to_mcq(qp->ibqp.send_cq);
1422 *recv_cq = *send_cq;
1423 break;
1424 default:
1425 *recv_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.recv_cq) :
1426 to_mcq(qp->ibwq.cq);
1427 *send_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.send_cq) :
1428 *recv_cq;
1429 break;
1430 }
1431}
1432
1433static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1434{
1435 if (qp->state != IB_QPS_RESET) {
1436 int i;
1437
1438 for (i = 0; i < (1 << qp->ibqp.rwq_ind_tbl->log_ind_tbl_size);
1439 i++) {
1440 struct ib_wq *ibwq = qp->ibqp.rwq_ind_tbl->ind_tbl[i];
1441 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
1442
1443 mutex_lock(&wq->mutex);
1444
1445 wq->rss_usecnt--;
1446
1447 mutex_unlock(&wq->mutex);
1448 }
1449
1450 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1451 MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1452 pr_warn("modify QP %06x to RESET failed.\n",
1453 qp->mqp.qpn);
1454 }
1455
1456 mlx4_qp_remove(dev->dev, &qp->mqp);
1457 mlx4_qp_free(dev->dev, &qp->mqp);
1458 mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1459 del_gid_entries(qp);
1460 kfree(qp->rss_ctx);
1461}
1462
1463static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
1464 enum mlx4_ib_source_type src, int is_user)
1465{
1466 struct mlx4_ib_cq *send_cq, *recv_cq;
1467 unsigned long flags;
1468
1469 if (qp->state != IB_QPS_RESET) {
1470 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1471 MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1472 pr_warn("modify QP %06x to RESET failed.\n",
1473 qp->mqp.qpn);
1474 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
1475 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
1476 qp->pri.smac = 0;
1477 qp->pri.smac_port = 0;
1478 }
1479 if (qp->alt.smac) {
1480 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
1481 qp->alt.smac = 0;
1482 }
1483 if (qp->pri.vid < 0x1000) {
1484 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
1485 qp->pri.vid = 0xFFFF;
1486 qp->pri.candidate_vid = 0xFFFF;
1487 qp->pri.update_vid = 0;
1488 }
1489 if (qp->alt.vid < 0x1000) {
1490 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
1491 qp->alt.vid = 0xFFFF;
1492 qp->alt.candidate_vid = 0xFFFF;
1493 qp->alt.update_vid = 0;
1494 }
1495 }
1496
1497 get_cqs(qp, src, &send_cq, &recv_cq);
1498
1499 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1500 mlx4_ib_lock_cqs(send_cq, recv_cq);
1501
1502
1503 list_del(&qp->qps_list);
1504 list_del(&qp->cq_send_list);
1505 list_del(&qp->cq_recv_list);
1506 if (!is_user) {
1507 __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1508 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1509 if (send_cq != recv_cq)
1510 __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
1511 }
1512
1513 mlx4_qp_remove(dev->dev, &qp->mqp);
1514
1515 mlx4_ib_unlock_cqs(send_cq, recv_cq);
1516 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1517
1518 mlx4_qp_free(dev->dev, &qp->mqp);
1519
1520 if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
1521 if (qp->flags & MLX4_IB_QP_NETIF)
1522 mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1523 else if (src == MLX4_IB_RWQ_SRC)
1524 mlx4_ib_release_wqn(to_mucontext(
1525 qp->ibwq.uobject->context), qp, 1);
1526 else
1527 mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1528 }
1529
1530 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1531
1532 if (is_user) {
1533 if (qp->rq.wqe_cnt) {
1534 struct mlx4_ib_ucontext *mcontext = !src ?
1535 to_mucontext(qp->ibqp.uobject->context) :
1536 to_mucontext(qp->ibwq.uobject->context);
1537 mlx4_ib_db_unmap_user(mcontext, &qp->db);
1538 }
1539 ib_umem_release(qp->umem);
1540 } else {
1541 kvfree(qp->sq.wrid);
1542 kvfree(qp->rq.wrid);
1543 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1544 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
1545 free_proxy_bufs(&dev->ib_dev, qp);
1546 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1547 if (qp->rq.wqe_cnt)
1548 mlx4_db_free(dev->dev, &qp->db);
1549 }
1550
1551 del_gid_entries(qp);
1552}
1553
1554static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
1555{
1556
1557 if (!mlx4_is_mfunc(dev->dev) ||
1558 (mlx4_is_master(dev->dev) &&
1559 attr->create_flags & MLX4_IB_SRIOV_SQP)) {
1560 return dev->dev->phys_caps.base_sqpn +
1561 (attr->qp_type == IB_QPT_SMI ? 0 : 2) +
1562 attr->port_num - 1;
1563 }
1564
1565 if (attr->qp_type == IB_QPT_SMI)
1566 return dev->dev->caps.spec_qps[attr->port_num - 1].qp0_proxy;
1567 else
1568 return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
1569}
1570
1571static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
1572 struct ib_qp_init_attr *init_attr,
1573 struct ib_udata *udata)
1574{
1575 struct mlx4_ib_qp *qp = NULL;
1576 int err;
1577 int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1578 u16 xrcdn = 0;
1579
1580 if (init_attr->rwq_ind_tbl)
1581 return _mlx4_ib_create_qp_rss(pd, init_attr, udata);
1582
1583
1584
1585
1586
1587 if (init_attr->create_flags & ~(MLX4_IB_QP_LSO |
1588 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK |
1589 MLX4_IB_SRIOV_TUNNEL_QP |
1590 MLX4_IB_SRIOV_SQP |
1591 MLX4_IB_QP_NETIF |
1592 MLX4_IB_QP_CREATE_ROCE_V2_GSI))
1593 return ERR_PTR(-EINVAL);
1594
1595 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1596 if (init_attr->qp_type != IB_QPT_UD)
1597 return ERR_PTR(-EINVAL);
1598 }
1599
1600 if (init_attr->create_flags) {
1601 if (udata && init_attr->create_flags & ~(sup_u_create_flags))
1602 return ERR_PTR(-EINVAL);
1603
1604 if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
1605 MLX4_IB_QP_CREATE_ROCE_V2_GSI |
1606 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) &&
1607 init_attr->qp_type != IB_QPT_UD) ||
1608 (init_attr->create_flags & MLX4_IB_SRIOV_SQP &&
1609 init_attr->qp_type > IB_QPT_GSI) ||
1610 (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
1611 init_attr->qp_type != IB_QPT_GSI))
1612 return ERR_PTR(-EINVAL);
1613 }
1614
1615 switch (init_attr->qp_type) {
1616 case IB_QPT_XRC_TGT:
1617 pd = to_mxrcd(init_attr->xrcd)->pd;
1618 xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1619 init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1620
1621 case IB_QPT_XRC_INI:
1622 if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1623 return ERR_PTR(-ENOSYS);
1624 init_attr->recv_cq = init_attr->send_cq;
1625
1626 case IB_QPT_RC:
1627 case IB_QPT_UC:
1628 case IB_QPT_RAW_PACKET:
1629 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1630 if (!qp)
1631 return ERR_PTR(-ENOMEM);
1632 qp->pri.vid = 0xFFFF;
1633 qp->alt.vid = 0xFFFF;
1634
1635 case IB_QPT_UD:
1636 {
1637 err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1638 init_attr, udata, 0, &qp);
1639 if (err) {
1640 kfree(qp);
1641 return ERR_PTR(err);
1642 }
1643
1644 qp->ibqp.qp_num = qp->mqp.qpn;
1645 qp->xrcdn = xrcdn;
1646
1647 break;
1648 }
1649 case IB_QPT_SMI:
1650 case IB_QPT_GSI:
1651 {
1652 int sqpn;
1653
1654
1655 if (udata)
1656 return ERR_PTR(-EINVAL);
1657 if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1658 int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1659 1, 1, &sqpn, 0,
1660 MLX4_RES_USAGE_DRIVER);
1661
1662 if (res)
1663 return ERR_PTR(res);
1664 } else {
1665 sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
1666 }
1667
1668 err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1669 init_attr, udata, sqpn, &qp);
1670 if (err)
1671 return ERR_PTR(err);
1672
1673 qp->port = init_attr->port_num;
1674 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
1675 init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
1676 break;
1677 }
1678 default:
1679
1680 return ERR_PTR(-EINVAL);
1681 }
1682
1683 return &qp->ibqp;
1684}
1685
1686struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
1687 struct ib_qp_init_attr *init_attr,
1688 struct ib_udata *udata) {
1689 struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
1690 struct ib_qp *ibqp;
1691 struct mlx4_ib_dev *dev = to_mdev(device);
1692
1693 ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);
1694
1695 if (!IS_ERR(ibqp) &&
1696 (init_attr->qp_type == IB_QPT_GSI) &&
1697 !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
1698 struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));
1699 int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
1700
1701 if (is_eth &&
1702 dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
1703 init_attr->create_flags |= MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1704 sqp->roce_v2_gsi = ib_create_qp(pd, init_attr);
1705
1706 if (IS_ERR(sqp->roce_v2_gsi)) {
1707 pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
1708 sqp->roce_v2_gsi = NULL;
1709 } else {
1710 sqp = to_msqp(to_mqp(sqp->roce_v2_gsi));
1711 sqp->qp.flags |= MLX4_IB_ROCE_V2_GSI_QP;
1712 }
1713
1714 init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1715 }
1716 }
1717 return ibqp;
1718}
1719
1720static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
1721{
1722 struct mlx4_ib_dev *dev = to_mdev(qp->device);
1723 struct mlx4_ib_qp *mqp = to_mqp(qp);
1724
1725 if (is_qp0(dev, mqp))
1726 mlx4_CLOSE_PORT(dev->dev, mqp->port);
1727
1728 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI &&
1729 dev->qp1_proxy[mqp->port - 1] == mqp) {
1730 mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]);
1731 dev->qp1_proxy[mqp->port - 1] = NULL;
1732 mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]);
1733 }
1734
1735 if (mqp->counter_index)
1736 mlx4_ib_free_qp_counter(dev, mqp);
1737
1738 if (qp->rwq_ind_tbl) {
1739 destroy_qp_rss(dev, mqp);
1740 } else {
1741 struct mlx4_ib_pd *pd;
1742
1743 pd = get_pd(mqp);
1744 destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, !!pd->ibpd.uobject);
1745 }
1746
1747 if (is_sqp(dev, mqp))
1748 kfree(to_msqp(mqp));
1749 else
1750 kfree(mqp);
1751
1752 return 0;
1753}
1754
1755int mlx4_ib_destroy_qp(struct ib_qp *qp)
1756{
1757 struct mlx4_ib_qp *mqp = to_mqp(qp);
1758
1759 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
1760 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
1761
1762 if (sqp->roce_v2_gsi)
1763 ib_destroy_qp(sqp->roce_v2_gsi);
1764 }
1765
1766 return _mlx4_ib_destroy_qp(qp);
1767}
1768
1769static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
1770{
1771 switch (type) {
1772 case MLX4_IB_QPT_RC: return MLX4_QP_ST_RC;
1773 case MLX4_IB_QPT_UC: return MLX4_QP_ST_UC;
1774 case MLX4_IB_QPT_UD: return MLX4_QP_ST_UD;
1775 case MLX4_IB_QPT_XRC_INI:
1776 case MLX4_IB_QPT_XRC_TGT: return MLX4_QP_ST_XRC;
1777 case MLX4_IB_QPT_SMI:
1778 case MLX4_IB_QPT_GSI:
1779 case MLX4_IB_QPT_RAW_PACKET: return MLX4_QP_ST_MLX;
1780
1781 case MLX4_IB_QPT_PROXY_SMI_OWNER:
1782 case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ?
1783 MLX4_QP_ST_MLX : -1);
1784 case MLX4_IB_QPT_PROXY_SMI:
1785 case MLX4_IB_QPT_TUN_SMI:
1786 case MLX4_IB_QPT_PROXY_GSI:
1787 case MLX4_IB_QPT_TUN_GSI: return (mlx4_is_mfunc(dev->dev) ?
1788 MLX4_QP_ST_UD : -1);
1789 default: return -1;
1790 }
1791}
1792
1793static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
1794 int attr_mask)
1795{
1796 u8 dest_rd_atomic;
1797 u32 access_flags;
1798 u32 hw_access_flags = 0;
1799
1800 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1801 dest_rd_atomic = attr->max_dest_rd_atomic;
1802 else
1803 dest_rd_atomic = qp->resp_depth;
1804
1805 if (attr_mask & IB_QP_ACCESS_FLAGS)
1806 access_flags = attr->qp_access_flags;
1807 else
1808 access_flags = qp->atomic_rd_en;
1809
1810 if (!dest_rd_atomic)
1811 access_flags &= IB_ACCESS_REMOTE_WRITE;
1812
1813 if (access_flags & IB_ACCESS_REMOTE_READ)
1814 hw_access_flags |= MLX4_QP_BIT_RRE;
1815 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
1816 hw_access_flags |= MLX4_QP_BIT_RAE;
1817 if (access_flags & IB_ACCESS_REMOTE_WRITE)
1818 hw_access_flags |= MLX4_QP_BIT_RWE;
1819
1820 return cpu_to_be32(hw_access_flags);
1821}
1822
1823static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
1824 int attr_mask)
1825{
1826 if (attr_mask & IB_QP_PKEY_INDEX)
1827 sqp->pkey_index = attr->pkey_index;
1828 if (attr_mask & IB_QP_QKEY)
1829 sqp->qkey = attr->qkey;
1830 if (attr_mask & IB_QP_SQ_PSN)
1831 sqp->send_psn = attr->sq_psn;
1832}
1833
1834static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
1835{
1836 path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
1837}
1838
1839static int _mlx4_set_path(struct mlx4_ib_dev *dev,
1840 const struct rdma_ah_attr *ah,
1841 u64 smac, u16 vlan_tag, struct mlx4_qp_path *path,
1842 struct mlx4_roce_smac_vlan_info *smac_info, u8 port)
1843{
1844 int vidx;
1845 int smac_index;
1846 int err;
1847
1848 path->grh_mylmc = rdma_ah_get_path_bits(ah) & 0x7f;
1849 path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
1850 if (rdma_ah_get_static_rate(ah)) {
1851 path->static_rate = rdma_ah_get_static_rate(ah) +
1852 MLX4_STAT_RATE_OFFSET;
1853 while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
1854 !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
1855 --path->static_rate;
1856 } else
1857 path->static_rate = 0;
1858
1859 if (rdma_ah_get_ah_flags(ah) & IB_AH_GRH) {
1860 const struct ib_global_route *grh = rdma_ah_read_grh(ah);
1861 int real_sgid_index =
1862 mlx4_ib_gid_index_to_real_index(dev, port,
1863 grh->sgid_index);
1864
1865 if (real_sgid_index < 0)
1866 return real_sgid_index;
1867 if (real_sgid_index >= dev->dev->caps.gid_table_len[port]) {
1868 pr_err("sgid_index (%u) too large. max is %d\n",
1869 real_sgid_index, dev->dev->caps.gid_table_len[port] - 1);
1870 return -1;
1871 }
1872
1873 path->grh_mylmc |= 1 << 7;
1874 path->mgid_index = real_sgid_index;
1875 path->hop_limit = grh->hop_limit;
1876 path->tclass_flowlabel =
1877 cpu_to_be32((grh->traffic_class << 20) |
1878 (grh->flow_label));
1879 memcpy(path->rgid, grh->dgid.raw, 16);
1880 }
1881
1882 if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
1883 if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH))
1884 return -1;
1885
1886 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1887 ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 7) << 3);
1888
1889 path->feup |= MLX4_FEUP_FORCE_ETH_UP;
1890 if (vlan_tag < 0x1000) {
1891 if (smac_info->vid < 0x1000) {
1892
1893 if (smac_info->vid != vlan_tag) {
1894
1895 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1896 if (err)
1897 return err;
1898 smac_info->candidate_vid = vlan_tag;
1899 smac_info->candidate_vlan_index = vidx;
1900 smac_info->candidate_vlan_port = port;
1901 smac_info->update_vid = 1;
1902 path->vlan_index = vidx;
1903 } else {
1904 path->vlan_index = smac_info->vlan_index;
1905 }
1906 } else {
1907
1908 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1909 if (err)
1910 return err;
1911 smac_info->candidate_vid = vlan_tag;
1912 smac_info->candidate_vlan_index = vidx;
1913 smac_info->candidate_vlan_port = port;
1914 smac_info->update_vid = 1;
1915 path->vlan_index = vidx;
1916 }
1917 path->feup |= MLX4_FVL_FORCE_ETH_VLAN;
1918 path->fl = 1 << 6;
1919 } else {
1920
1921 if (smac_info->vid < 0x1000) {
1922 smac_info->candidate_vid = 0xFFFF;
1923 smac_info->update_vid = 1;
1924 }
1925 }
1926
1927
1928
1929
1930
1931
1932 if ((!smac_info->smac && !smac_info->smac_port) ||
1933 smac_info->smac != smac) {
1934
1935 smac_index = mlx4_register_mac(dev->dev, port, smac);
1936 if (smac_index >= 0) {
1937 smac_info->candidate_smac_index = smac_index;
1938 smac_info->candidate_smac = smac;
1939 smac_info->candidate_smac_port = port;
1940 } else {
1941 return -EINVAL;
1942 }
1943 } else {
1944 smac_index = smac_info->smac_index;
1945 }
1946 memcpy(path->dmac, ah->roce.dmac, 6);
1947 path->ackto = MLX4_IB_LINK_TYPE_ETH;
1948
1949 path->grh_mylmc = (u8) (smac_index) | 0x80;
1950 } else {
1951 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1952 ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 0xf) << 2);
1953 }
1954
1955 return 0;
1956}
1957
1958static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp,
1959 enum ib_qp_attr_mask qp_attr_mask,
1960 struct mlx4_ib_qp *mqp,
1961 struct mlx4_qp_path *path, u8 port,
1962 u16 vlan_id, u8 *smac)
1963{
1964 return _mlx4_set_path(dev, &qp->ah_attr,
1965 mlx4_mac_to_u64(smac),
1966 vlan_id,
1967 path, &mqp->pri, port);
1968}
1969
1970static int mlx4_set_alt_path(struct mlx4_ib_dev *dev,
1971 const struct ib_qp_attr *qp,
1972 enum ib_qp_attr_mask qp_attr_mask,
1973 struct mlx4_ib_qp *mqp,
1974 struct mlx4_qp_path *path, u8 port)
1975{
1976 return _mlx4_set_path(dev, &qp->alt_ah_attr,
1977 0,
1978 0xffff,
1979 path, &mqp->alt, port);
1980}
1981
1982static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1983{
1984 struct mlx4_ib_gid_entry *ge, *tmp;
1985
1986 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1987 if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
1988 ge->added = 1;
1989 ge->port = qp->port;
1990 }
1991 }
1992}
1993
1994static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev,
1995 struct mlx4_ib_qp *qp,
1996 struct mlx4_qp_context *context)
1997{
1998 u64 u64_mac;
1999 int smac_index;
2000
2001 u64_mac = atomic64_read(&dev->iboe.mac[qp->port - 1]);
2002
2003 context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6);
2004 if (!qp->pri.smac && !qp->pri.smac_port) {
2005 smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac);
2006 if (smac_index >= 0) {
2007 qp->pri.candidate_smac_index = smac_index;
2008 qp->pri.candidate_smac = u64_mac;
2009 qp->pri.candidate_smac_port = qp->port;
2010 context->pri_path.grh_mylmc = 0x80 | (u8) smac_index;
2011 } else {
2012 return -ENOENT;
2013 }
2014 }
2015 return 0;
2016}
2017
2018static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
2019{
2020 struct counter_index *new_counter_index;
2021 int err;
2022 u32 tmp_idx;
2023
2024 if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) !=
2025 IB_LINK_LAYER_ETHERNET ||
2026 !(qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) ||
2027 !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
2028 return 0;
2029
2030 err = mlx4_counter_alloc(dev->dev, &tmp_idx, MLX4_RES_USAGE_DRIVER);
2031 if (err)
2032 return err;
2033
2034 new_counter_index = kmalloc(sizeof(*new_counter_index), GFP_KERNEL);
2035 if (!new_counter_index) {
2036 mlx4_counter_free(dev->dev, tmp_idx);
2037 return -ENOMEM;
2038 }
2039
2040 new_counter_index->index = tmp_idx;
2041 new_counter_index->allocated = 1;
2042 qp->counter_index = new_counter_index;
2043
2044 mutex_lock(&dev->counters_table[qp->port - 1].mutex);
2045 list_add_tail(&new_counter_index->list,
2046 &dev->counters_table[qp->port - 1].counters_list);
2047 mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
2048
2049 return 0;
2050}
2051
2052enum {
2053 MLX4_QPC_ROCE_MODE_1 = 0,
2054 MLX4_QPC_ROCE_MODE_2 = 2,
2055 MLX4_QPC_ROCE_MODE_UNDEFINED = 0xff
2056};
2057
2058static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
2059{
2060 switch (gid_type) {
2061 case IB_GID_TYPE_ROCE:
2062 return MLX4_QPC_ROCE_MODE_1;
2063 case IB_GID_TYPE_ROCE_UDP_ENCAP:
2064 return MLX4_QPC_ROCE_MODE_2;
2065 default:
2066 return MLX4_QPC_ROCE_MODE_UNDEFINED;
2067 }
2068}
2069
2070
2071
2072
2073
2074static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num)
2075{
2076 int err = 0;
2077 int i;
2078
2079 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
2080 struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
2081 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2082
2083 mutex_lock(&wq->mutex);
2084
2085
2086
2087
2088
2089
2090
2091 if ((wq->rss_usecnt > 0) && (wq->port != port_num)) {
2092 err = -EINVAL;
2093 mutex_unlock(&wq->mutex);
2094 break;
2095 }
2096 wq->port = port_num;
2097 if ((wq->rss_usecnt == 0) && (ibwq->state == IB_WQS_RDY)) {
2098 err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY);
2099 if (err) {
2100 mutex_unlock(&wq->mutex);
2101 break;
2102 }
2103 }
2104 wq->rss_usecnt++;
2105
2106 mutex_unlock(&wq->mutex);
2107 }
2108
2109 if (i && err) {
2110 int j;
2111
2112 for (j = (i - 1); j >= 0; j--) {
2113 struct ib_wq *ibwq = ind_tbl->ind_tbl[j];
2114 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2115
2116 mutex_lock(&wq->mutex);
2117
2118 if ((wq->rss_usecnt == 1) &&
2119 (ibwq->state == IB_WQS_RDY))
2120 if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
2121 pr_warn("failed to reverse WQN=0x%06x\n",
2122 ibwq->wq_num);
2123 wq->rss_usecnt--;
2124
2125 mutex_unlock(&wq->mutex);
2126 }
2127 }
2128
2129 return err;
2130}
2131
2132static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl)
2133{
2134 int i;
2135
2136 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
2137 struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
2138 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2139
2140 mutex_lock(&wq->mutex);
2141
2142 if ((wq->rss_usecnt == 1) && (ibwq->state == IB_WQS_RDY))
2143 if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
2144 pr_warn("failed to reverse WQN=%x\n",
2145 ibwq->wq_num);
2146 wq->rss_usecnt--;
2147
2148 mutex_unlock(&wq->mutex);
2149 }
2150}
2151
2152static void fill_qp_rss_context(struct mlx4_qp_context *context,
2153 struct mlx4_ib_qp *qp)
2154{
2155 struct mlx4_rss_context *rss_context;
2156
2157 rss_context = (void *)context + offsetof(struct mlx4_qp_context,
2158 pri_path) + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
2159
2160 rss_context->base_qpn = cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz);
2161 rss_context->default_qpn =
2162 cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz & 0xffffff);
2163 if (qp->rss_ctx->flags & (MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6))
2164 rss_context->base_qpn_udp = rss_context->default_qpn;
2165 rss_context->flags = qp->rss_ctx->flags;
2166
2167 rss_context->hash_fn = MLX4_RSS_HASH_TOP;
2168
2169 memcpy(rss_context->rss_key, qp->rss_ctx->rss_key,
2170 MLX4_EN_RSS_KEY_SIZE);
2171}
2172
2173static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
2174 const struct ib_qp_attr *attr, int attr_mask,
2175 enum ib_qp_state cur_state, enum ib_qp_state new_state)
2176{
2177 struct ib_uobject *ibuobject;
2178 struct ib_srq *ibsrq;
2179 struct ib_rwq_ind_table *rwq_ind_tbl;
2180 enum ib_qp_type qp_type;
2181 struct mlx4_ib_dev *dev;
2182 struct mlx4_ib_qp *qp;
2183 struct mlx4_ib_pd *pd;
2184 struct mlx4_ib_cq *send_cq, *recv_cq;
2185 struct mlx4_qp_context *context;
2186 enum mlx4_qp_optpar optpar = 0;
2187 int sqd_event;
2188 int steer_qp = 0;
2189 int err = -EINVAL;
2190 int counter_index;
2191
2192 if (src_type == MLX4_IB_RWQ_SRC) {
2193 struct ib_wq *ibwq;
2194
2195 ibwq = (struct ib_wq *)src;
2196 ibuobject = ibwq->uobject;
2197 ibsrq = NULL;
2198 rwq_ind_tbl = NULL;
2199 qp_type = IB_QPT_RAW_PACKET;
2200 qp = to_mqp((struct ib_qp *)ibwq);
2201 dev = to_mdev(ibwq->device);
2202 pd = to_mpd(ibwq->pd);
2203 } else {
2204 struct ib_qp *ibqp;
2205
2206 ibqp = (struct ib_qp *)src;
2207 ibuobject = ibqp->uobject;
2208 ibsrq = ibqp->srq;
2209 rwq_ind_tbl = ibqp->rwq_ind_tbl;
2210 qp_type = ibqp->qp_type;
2211 qp = to_mqp(ibqp);
2212 dev = to_mdev(ibqp->device);
2213 pd = get_pd(qp);
2214 }
2215
2216
2217 if (attr_mask & IB_QP_ALT_PATH &&
2218 rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2219 IB_LINK_LAYER_ETHERNET)
2220 return -ENOTSUPP;
2221
2222 context = kzalloc(sizeof *context, GFP_KERNEL);
2223 if (!context)
2224 return -ENOMEM;
2225
2226 context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
2227 (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16));
2228
2229 if (!(attr_mask & IB_QP_PATH_MIG_STATE))
2230 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2231 else {
2232 optpar |= MLX4_QP_OPTPAR_PM_STATE;
2233 switch (attr->path_mig_state) {
2234 case IB_MIG_MIGRATED:
2235 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2236 break;
2237 case IB_MIG_REARM:
2238 context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
2239 break;
2240 case IB_MIG_ARMED:
2241 context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
2242 break;
2243 }
2244 }
2245
2246 if (qp->inl_recv_sz)
2247 context->param3 |= cpu_to_be32(1 << 25);
2248
2249 if (qp->flags & MLX4_IB_QP_SCATTER_FCS)
2250 context->param3 |= cpu_to_be32(1 << 29);
2251
2252 if (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI)
2253 context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
2254 else if (qp_type == IB_QPT_RAW_PACKET)
2255 context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
2256 else if (qp_type == IB_QPT_UD) {
2257 if (qp->flags & MLX4_IB_QP_LSO)
2258 context->mtu_msgmax = (IB_MTU_4096 << 5) |
2259 ilog2(dev->dev->caps.max_gso_sz);
2260 else
2261 context->mtu_msgmax = (IB_MTU_4096 << 5) | 13;
2262 } else if (attr_mask & IB_QP_PATH_MTU) {
2263 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
2264 pr_err("path MTU (%u) is invalid\n",
2265 attr->path_mtu);
2266 goto out;
2267 }
2268 context->mtu_msgmax = (attr->path_mtu << 5) |
2269 ilog2(dev->dev->caps.max_msg_sz);
2270 }
2271
2272 if (!rwq_ind_tbl) {
2273 if (qp->rq.wqe_cnt)
2274 context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
2275 context->rq_size_stride |= qp->rq.wqe_shift - 4;
2276 }
2277
2278 if (qp->sq.wqe_cnt)
2279 context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
2280 context->sq_size_stride |= qp->sq.wqe_shift - 4;
2281
2282 if (new_state == IB_QPS_RESET && qp->counter_index)
2283 mlx4_ib_free_qp_counter(dev, qp);
2284
2285 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2286 context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
2287 context->xrcd = cpu_to_be32((u32) qp->xrcdn);
2288 if (qp_type == IB_QPT_RAW_PACKET)
2289 context->param3 |= cpu_to_be32(1 << 30);
2290 }
2291
2292 if (ibuobject)
2293 context->usr_page = cpu_to_be32(
2294 mlx4_to_hw_uar_index(dev->dev,
2295 to_mucontext(ibuobject->context)
2296 ->uar.index));
2297 else
2298 context->usr_page = cpu_to_be32(
2299 mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
2300
2301 if (attr_mask & IB_QP_DEST_QPN)
2302 context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
2303
2304 if (attr_mask & IB_QP_PORT) {
2305 if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
2306 !(attr_mask & IB_QP_AV)) {
2307 mlx4_set_sched(&context->pri_path, attr->port_num);
2308 optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
2309 }
2310 }
2311
2312 if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2313 err = create_qp_lb_counter(dev, qp);
2314 if (err)
2315 goto out;
2316
2317 counter_index =
2318 dev->counters_table[qp->port - 1].default_counter;
2319 if (qp->counter_index)
2320 counter_index = qp->counter_index->index;
2321
2322 if (counter_index != -1) {
2323 context->pri_path.counter_index = counter_index;
2324 optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX;
2325 if (qp->counter_index) {
2326 context->pri_path.fl |=
2327 MLX4_FL_ETH_SRC_CHECK_MC_LB;
2328 context->pri_path.vlan_control |=
2329 MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER;
2330 }
2331 } else
2332 context->pri_path.counter_index =
2333 MLX4_SINK_COUNTER_INDEX(dev->dev);
2334
2335 if (qp->flags & MLX4_IB_QP_NETIF) {
2336 mlx4_ib_steer_qp_reg(dev, qp, 1);
2337 steer_qp = 1;
2338 }
2339
2340 if (qp_type == IB_QPT_GSI) {
2341 enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ?
2342 IB_GID_TYPE_ROCE_UDP_ENCAP : IB_GID_TYPE_ROCE;
2343 u8 qpc_roce_mode = gid_type_to_qpc(gid_type);
2344
2345 context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2346 }
2347 }
2348
2349 if (attr_mask & IB_QP_PKEY_INDEX) {
2350 if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2351 context->pri_path.disable_pkey_check = 0x40;
2352 context->pri_path.pkey_index = attr->pkey_index;
2353 optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
2354 }
2355
2356 if (attr_mask & IB_QP_AV) {
2357 u8 port_num = mlx4_is_bonded(dev->dev) ? 1 :
2358 attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2359 union ib_gid gid;
2360 struct ib_gid_attr gid_attr = {.gid_type = IB_GID_TYPE_IB};
2361 u16 vlan = 0xffff;
2362 u8 smac[ETH_ALEN];
2363 int status = 0;
2364 int is_eth =
2365 rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
2366 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
2367
2368 if (is_eth) {
2369 int index =
2370 rdma_ah_read_grh(&attr->ah_attr)->sgid_index;
2371
2372 status = ib_get_cached_gid(&dev->ib_dev, port_num,
2373 index, &gid, &gid_attr);
2374 if (!status) {
2375 vlan = rdma_vlan_dev_vlan_id(gid_attr.ndev);
2376 memcpy(smac, gid_attr.ndev->dev_addr, ETH_ALEN);
2377 dev_put(gid_attr.ndev);
2378 }
2379 }
2380 if (status)
2381 goto out;
2382
2383 if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
2384 port_num, vlan, smac))
2385 goto out;
2386
2387 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
2388 MLX4_QP_OPTPAR_SCHED_QUEUE);
2389
2390 if (is_eth &&
2391 (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)) {
2392 u8 qpc_roce_mode = gid_type_to_qpc(gid_attr.gid_type);
2393
2394 if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_UNDEFINED) {
2395 err = -EINVAL;
2396 goto out;
2397 }
2398 context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2399 }
2400
2401 }
2402
2403 if (attr_mask & IB_QP_TIMEOUT) {
2404 context->pri_path.ackto |= attr->timeout << 3;
2405 optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
2406 }
2407
2408 if (attr_mask & IB_QP_ALT_PATH) {
2409 if (attr->alt_port_num == 0 ||
2410 attr->alt_port_num > dev->dev->caps.num_ports)
2411 goto out;
2412
2413 if (attr->alt_pkey_index >=
2414 dev->dev->caps.pkey_table_len[attr->alt_port_num])
2415 goto out;
2416
2417 if (mlx4_set_alt_path(dev, attr, attr_mask, qp,
2418 &context->alt_path,
2419 attr->alt_port_num))
2420 goto out;
2421
2422 context->alt_path.pkey_index = attr->alt_pkey_index;
2423 context->alt_path.ackto = attr->alt_timeout << 3;
2424 optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
2425 }
2426
2427 context->pd = cpu_to_be32(pd->pdn);
2428
2429 if (!rwq_ind_tbl) {
2430 context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
2431 get_cqs(qp, src_type, &send_cq, &recv_cq);
2432 } else {
2433 send_cq = to_mcq(rwq_ind_tbl->ind_tbl[0]->cq);
2434 recv_cq = send_cq;
2435 }
2436 context->cqn_send = cpu_to_be32(send_cq->mcq.cqn);
2437 context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
2438
2439
2440 if (!ibuobject)
2441 context->params1 |= cpu_to_be32(1 << 11);
2442
2443 if (attr_mask & IB_QP_RNR_RETRY) {
2444 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
2445 optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
2446 }
2447
2448 if (attr_mask & IB_QP_RETRY_CNT) {
2449 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
2450 optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
2451 }
2452
2453 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2454 if (attr->max_rd_atomic)
2455 context->params1 |=
2456 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
2457 optpar |= MLX4_QP_OPTPAR_SRA_MAX;
2458 }
2459
2460 if (attr_mask & IB_QP_SQ_PSN)
2461 context->next_send_psn = cpu_to_be32(attr->sq_psn);
2462
2463 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2464 if (attr->max_dest_rd_atomic)
2465 context->params2 |=
2466 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
2467 optpar |= MLX4_QP_OPTPAR_RRA_MAX;
2468 }
2469
2470 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
2471 context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
2472 optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
2473 }
2474
2475 if (ibsrq)
2476 context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
2477
2478 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2479 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
2480 optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
2481 }
2482 if (attr_mask & IB_QP_RQ_PSN)
2483 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
2484
2485
2486 if (attr_mask & IB_QP_QKEY) {
2487 if (qp->mlx4_ib_qp_type &
2488 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))
2489 context->qkey = cpu_to_be32(IB_QP_SET_QKEY);
2490 else {
2491 if (mlx4_is_mfunc(dev->dev) &&
2492 !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) &&
2493 (attr->qkey & MLX4_RESERVED_QKEY_MASK) ==
2494 MLX4_RESERVED_QKEY_BASE) {
2495 pr_err("Cannot use reserved QKEY"
2496 " 0x%x (range 0xffff0000..0xffffffff"
2497 " is reserved)\n", attr->qkey);
2498 err = -EINVAL;
2499 goto out;
2500 }
2501 context->qkey = cpu_to_be32(attr->qkey);
2502 }
2503 optpar |= MLX4_QP_OPTPAR_Q_KEY;
2504 }
2505
2506 if (ibsrq)
2507 context->srqn = cpu_to_be32(1 << 24 |
2508 to_msrq(ibsrq)->msrq.srqn);
2509
2510 if (qp->rq.wqe_cnt &&
2511 cur_state == IB_QPS_RESET &&
2512 new_state == IB_QPS_INIT)
2513 context->db_rec_addr = cpu_to_be64(qp->db.dma);
2514
2515 if (cur_state == IB_QPS_INIT &&
2516 new_state == IB_QPS_RTR &&
2517 (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI ||
2518 qp_type == IB_QPT_UD || qp_type == IB_QPT_RAW_PACKET)) {
2519 context->pri_path.sched_queue = (qp->port - 1) << 6;
2520 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
2521 qp->mlx4_ib_qp_type &
2522 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) {
2523 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
2524 if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI)
2525 context->pri_path.fl = 0x80;
2526 } else {
2527 if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2528 context->pri_path.fl = 0x80;
2529 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
2530 }
2531 if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2532 IB_LINK_LAYER_ETHERNET) {
2533 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI ||
2534 qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI)
2535 context->pri_path.feup = 1 << 7;
2536
2537 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD ||
2538 qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI ||
2539 qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) {
2540 err = handle_eth_ud_smac_index(dev, qp, context);
2541 if (err) {
2542 err = -EINVAL;
2543 goto out;
2544 }
2545 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
2546 dev->qp1_proxy[qp->port - 1] = qp;
2547 }
2548 }
2549 }
2550
2551 if (qp_type == IB_QPT_RAW_PACKET) {
2552 context->pri_path.ackto = (context->pri_path.ackto & 0xf8) |
2553 MLX4_IB_LINK_TYPE_ETH;
2554 if (dev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
2555
2556 if (!rwq_ind_tbl)
2557 context->srqn = cpu_to_be32(7 << 28);
2558 }
2559 }
2560
2561 if (qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
2562 int is_eth = rdma_port_get_link_layer(
2563 &dev->ib_dev, qp->port) ==
2564 IB_LINK_LAYER_ETHERNET;
2565 if (is_eth) {
2566 context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH;
2567 optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH;
2568 }
2569 }
2570
2571 if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD &&
2572 attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
2573 sqd_event = 1;
2574 else
2575 sqd_event = 0;
2576
2577 if (!ibuobject &&
2578 cur_state == IB_QPS_RESET &&
2579 new_state == IB_QPS_INIT)
2580 context->rlkey_roce_mode |= (1 << 4);
2581
2582
2583
2584
2585
2586
2587
2588 if (!ibuobject &&
2589 cur_state == IB_QPS_RESET &&
2590 new_state == IB_QPS_INIT) {
2591 struct mlx4_wqe_ctrl_seg *ctrl;
2592 int i;
2593
2594 for (i = 0; i < qp->sq.wqe_cnt; ++i) {
2595 ctrl = get_send_wqe(qp, i);
2596 ctrl->owner_opcode = cpu_to_be32(1 << 31);
2597 if (qp->sq_max_wqes_per_wr == 1)
2598 ctrl->qpn_vlan.fence_size =
2599 1 << (qp->sq.wqe_shift - 4);
2600
2601 stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
2602 }
2603 }
2604
2605 if (rwq_ind_tbl &&
2606 cur_state == IB_QPS_RESET &&
2607 new_state == IB_QPS_INIT) {
2608 fill_qp_rss_context(context, qp);
2609 context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET);
2610 }
2611
2612 err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
2613 to_mlx4_state(new_state), context, optpar,
2614 sqd_event, &qp->mqp);
2615 if (err)
2616 goto out;
2617
2618 qp->state = new_state;
2619
2620 if (attr_mask & IB_QP_ACCESS_FLAGS)
2621 qp->atomic_rd_en = attr->qp_access_flags;
2622 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2623 qp->resp_depth = attr->max_dest_rd_atomic;
2624 if (attr_mask & IB_QP_PORT) {
2625 qp->port = attr->port_num;
2626 update_mcg_macs(dev, qp);
2627 }
2628 if (attr_mask & IB_QP_ALT_PATH)
2629 qp->alt_port = attr->alt_port_num;
2630
2631 if (is_sqp(dev, qp))
2632 store_sqp_attrs(to_msqp(qp), attr, attr_mask);
2633
2634
2635
2636
2637
2638 if (is_qp0(dev, qp)) {
2639 if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
2640 if (mlx4_INIT_PORT(dev->dev, qp->port))
2641 pr_warn("INIT_PORT failed for port %d\n",
2642 qp->port);
2643
2644 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
2645 (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
2646 mlx4_CLOSE_PORT(dev->dev, qp->port);
2647 }
2648
2649
2650
2651
2652
2653 if (new_state == IB_QPS_RESET) {
2654 if (!ibuobject) {
2655 mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
2656 ibsrq ? to_msrq(ibsrq) : NULL);
2657 if (send_cq != recv_cq)
2658 mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
2659
2660 qp->rq.head = 0;
2661 qp->rq.tail = 0;
2662 qp->sq.head = 0;
2663 qp->sq.tail = 0;
2664 qp->sq_next_wqe = 0;
2665 if (qp->rq.wqe_cnt)
2666 *qp->db.db = 0;
2667
2668 if (qp->flags & MLX4_IB_QP_NETIF)
2669 mlx4_ib_steer_qp_reg(dev, qp, 0);
2670 }
2671 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
2672 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2673 qp->pri.smac = 0;
2674 qp->pri.smac_port = 0;
2675 }
2676 if (qp->alt.smac) {
2677 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2678 qp->alt.smac = 0;
2679 }
2680 if (qp->pri.vid < 0x1000) {
2681 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
2682 qp->pri.vid = 0xFFFF;
2683 qp->pri.candidate_vid = 0xFFFF;
2684 qp->pri.update_vid = 0;
2685 }
2686
2687 if (qp->alt.vid < 0x1000) {
2688 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
2689 qp->alt.vid = 0xFFFF;
2690 qp->alt.candidate_vid = 0xFFFF;
2691 qp->alt.update_vid = 0;
2692 }
2693 }
2694out:
2695 if (err && qp->counter_index)
2696 mlx4_ib_free_qp_counter(dev, qp);
2697 if (err && steer_qp)
2698 mlx4_ib_steer_qp_reg(dev, qp, 0);
2699 kfree(context);
2700 if (qp->pri.candidate_smac ||
2701 (!qp->pri.candidate_smac && qp->pri.candidate_smac_port)) {
2702 if (err) {
2703 mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac);
2704 } else {
2705 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port))
2706 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2707 qp->pri.smac = qp->pri.candidate_smac;
2708 qp->pri.smac_index = qp->pri.candidate_smac_index;
2709 qp->pri.smac_port = qp->pri.candidate_smac_port;
2710 }
2711 qp->pri.candidate_smac = 0;
2712 qp->pri.candidate_smac_index = 0;
2713 qp->pri.candidate_smac_port = 0;
2714 }
2715 if (qp->alt.candidate_smac) {
2716 if (err) {
2717 mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->alt.candidate_smac);
2718 } else {
2719 if (qp->alt.smac)
2720 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2721 qp->alt.smac = qp->alt.candidate_smac;
2722 qp->alt.smac_index = qp->alt.candidate_smac_index;
2723 qp->alt.smac_port = qp->alt.candidate_smac_port;
2724 }
2725 qp->alt.candidate_smac = 0;
2726 qp->alt.candidate_smac_index = 0;
2727 qp->alt.candidate_smac_port = 0;
2728 }
2729
2730 if (qp->pri.update_vid) {
2731 if (err) {
2732 if (qp->pri.candidate_vid < 0x1000)
2733 mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port,
2734 qp->pri.candidate_vid);
2735 } else {
2736 if (qp->pri.vid < 0x1000)
2737 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port,
2738 qp->pri.vid);
2739 qp->pri.vid = qp->pri.candidate_vid;
2740 qp->pri.vlan_port = qp->pri.candidate_vlan_port;
2741 qp->pri.vlan_index = qp->pri.candidate_vlan_index;
2742 }
2743 qp->pri.candidate_vid = 0xFFFF;
2744 qp->pri.update_vid = 0;
2745 }
2746
2747 if (qp->alt.update_vid) {
2748 if (err) {
2749 if (qp->alt.candidate_vid < 0x1000)
2750 mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port,
2751 qp->alt.candidate_vid);
2752 } else {
2753 if (qp->alt.vid < 0x1000)
2754 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port,
2755 qp->alt.vid);
2756 qp->alt.vid = qp->alt.candidate_vid;
2757 qp->alt.vlan_port = qp->alt.candidate_vlan_port;
2758 qp->alt.vlan_index = qp->alt.candidate_vlan_index;
2759 }
2760 qp->alt.candidate_vid = 0xFFFF;
2761 qp->alt.update_vid = 0;
2762 }
2763
2764 return err;
2765}
2766
2767enum {
2768 MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK = (IB_QP_STATE |
2769 IB_QP_PORT),
2770};
2771
2772static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2773 int attr_mask, struct ib_udata *udata)
2774{
2775 enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
2776 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2777 struct mlx4_ib_qp *qp = to_mqp(ibqp);
2778 enum ib_qp_state cur_state, new_state;
2779 int err = -EINVAL;
2780 mutex_lock(&qp->mutex);
2781
2782 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2783 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2784
2785 if (cur_state != new_state || cur_state != IB_QPS_RESET) {
2786 int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2787 ll = rdma_port_get_link_layer(&dev->ib_dev, port);
2788 }
2789
2790 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
2791 attr_mask, ll)) {
2792 pr_debug("qpn 0x%x: invalid attribute mask specified "
2793 "for transition %d to %d. qp_type %d,"
2794 " attr_mask 0x%x\n",
2795 ibqp->qp_num, cur_state, new_state,
2796 ibqp->qp_type, attr_mask);
2797 goto out;
2798 }
2799
2800 if (ibqp->rwq_ind_tbl) {
2801 if (!(((cur_state == IB_QPS_RESET) &&
2802 (new_state == IB_QPS_INIT)) ||
2803 ((cur_state == IB_QPS_INIT) &&
2804 (new_state == IB_QPS_RTR)))) {
2805 pr_debug("qpn 0x%x: RSS QP unsupported transition %d to %d\n",
2806 ibqp->qp_num, cur_state, new_state);
2807
2808 err = -EOPNOTSUPP;
2809 goto out;
2810 }
2811
2812 if (attr_mask & ~MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK) {
2813 pr_debug("qpn 0x%x: RSS QP unsupported attribute mask 0x%x for transition %d to %d\n",
2814 ibqp->qp_num, attr_mask, cur_state, new_state);
2815
2816 err = -EOPNOTSUPP;
2817 goto out;
2818 }
2819 }
2820
2821 if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT)) {
2822 if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
2823 if ((ibqp->qp_type == IB_QPT_RC) ||
2824 (ibqp->qp_type == IB_QPT_UD) ||
2825 (ibqp->qp_type == IB_QPT_UC) ||
2826 (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
2827 (ibqp->qp_type == IB_QPT_XRC_INI)) {
2828 attr->port_num = mlx4_ib_bond_next_port(dev);
2829 }
2830 } else {
2831
2832
2833 attr_mask &= ~IB_QP_PORT;
2834 }
2835 }
2836
2837 if ((attr_mask & IB_QP_PORT) &&
2838 (attr->port_num == 0 || attr->port_num > dev->num_ports)) {
2839 pr_debug("qpn 0x%x: invalid port number (%d) specified "
2840 "for transition %d to %d. qp_type %d\n",
2841 ibqp->qp_num, attr->port_num, cur_state,
2842 new_state, ibqp->qp_type);
2843 goto out;
2844 }
2845
2846 if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
2847 (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) !=
2848 IB_LINK_LAYER_ETHERNET))
2849 goto out;
2850
2851 if (attr_mask & IB_QP_PKEY_INDEX) {
2852 int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2853 if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) {
2854 pr_debug("qpn 0x%x: invalid pkey index (%d) specified "
2855 "for transition %d to %d. qp_type %d\n",
2856 ibqp->qp_num, attr->pkey_index, cur_state,
2857 new_state, ibqp->qp_type);
2858 goto out;
2859 }
2860 }
2861
2862 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
2863 attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
2864 pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. "
2865 "Transition %d to %d. qp_type %d\n",
2866 ibqp->qp_num, attr->max_rd_atomic, cur_state,
2867 new_state, ibqp->qp_type);
2868 goto out;
2869 }
2870
2871 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
2872 attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
2873 pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. "
2874 "Transition %d to %d. qp_type %d\n",
2875 ibqp->qp_num, attr->max_dest_rd_atomic, cur_state,
2876 new_state, ibqp->qp_type);
2877 goto out;
2878 }
2879
2880 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
2881 err = 0;
2882 goto out;
2883 }
2884
2885 if (ibqp->rwq_ind_tbl && (new_state == IB_QPS_INIT)) {
2886 err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num);
2887 if (err)
2888 goto out;
2889 }
2890
2891 err = __mlx4_ib_modify_qp(ibqp, MLX4_IB_QP_SRC, attr, attr_mask,
2892 cur_state, new_state);
2893
2894 if (ibqp->rwq_ind_tbl && err)
2895 bring_down_rss_rwqs(ibqp->rwq_ind_tbl);
2896
2897 if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
2898 attr->port_num = 1;
2899
2900out:
2901 mutex_unlock(&qp->mutex);
2902 return err;
2903}
2904
2905int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2906 int attr_mask, struct ib_udata *udata)
2907{
2908 struct mlx4_ib_qp *mqp = to_mqp(ibqp);
2909 int ret;
2910
2911 ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
2912
2913 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2914 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
2915 int err = 0;
2916
2917 if (sqp->roce_v2_gsi)
2918 err = ib_modify_qp(sqp->roce_v2_gsi, attr, attr_mask);
2919 if (err)
2920 pr_err("Failed to modify GSI QP for RoCEv2 (%d)\n",
2921 err);
2922 }
2923 return ret;
2924}
2925
2926static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
2927{
2928 int i;
2929 for (i = 0; i < dev->caps.num_ports; i++) {
2930 if (qpn == dev->caps.spec_qps[i].qp0_proxy ||
2931 qpn == dev->caps.spec_qps[i].qp0_tunnel) {
2932 *qkey = dev->caps.spec_qps[i].qp0_qkey;
2933 return 0;
2934 }
2935 }
2936 return -EINVAL;
2937}
2938
2939static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
2940 struct ib_ud_wr *wr,
2941 void *wqe, unsigned *mlx_seg_len)
2942{
2943 struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device);
2944 struct ib_device *ib_dev = &mdev->ib_dev;
2945 struct mlx4_wqe_mlx_seg *mlx = wqe;
2946 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2947 struct mlx4_ib_ah *ah = to_mah(wr->ah);
2948 u16 pkey;
2949 u32 qkey;
2950 int send_size;
2951 int header_size;
2952 int spc;
2953 int i;
2954
2955 if (wr->wr.opcode != IB_WR_SEND)
2956 return -EINVAL;
2957
2958 send_size = 0;
2959
2960 for (i = 0; i < wr->wr.num_sge; ++i)
2961 send_size += wr->wr.sg_list[i].length;
2962
2963
2964
2965 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2966 send_size += sizeof (struct mlx4_ib_tunnel_header);
2967
2968 ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
2969
2970 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2971 sqp->ud_header.lrh.service_level =
2972 be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2973 sqp->ud_header.lrh.destination_lid =
2974 cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2975 sqp->ud_header.lrh.source_lid =
2976 cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2977 }
2978
2979 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2980
2981
2982 mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR);
2983 mlx->rlid = sqp->ud_header.lrh.destination_lid;
2984
2985 sqp->ud_header.lrh.virtual_lane = 0;
2986 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2987 ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey);
2988 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2989 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2990 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2991 else
2992 sqp->ud_header.bth.destination_qpn =
2993 cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel);
2994
2995 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2996 if (mlx4_is_master(mdev->dev)) {
2997 if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2998 return -EINVAL;
2999 } else {
3000 if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
3001 return -EINVAL;
3002 }
3003 sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
3004 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
3005
3006 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
3007 sqp->ud_header.immediate_present = 0;
3008
3009 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3010
3011
3012
3013
3014
3015
3016
3017 spc = MLX4_INLINE_ALIGN -
3018 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3019 if (header_size <= spc) {
3020 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
3021 memcpy(inl + 1, sqp->header_buf, header_size);
3022 i = 1;
3023 } else {
3024 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3025 memcpy(inl + 1, sqp->header_buf, spc);
3026
3027 inl = (void *) (inl + 1) + spc;
3028 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042 wmb();
3043 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
3044 i = 2;
3045 }
3046
3047 *mlx_seg_len =
3048 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
3049 return 0;
3050}
3051
3052static u8 sl_to_vl(struct mlx4_ib_dev *dev, u8 sl, int port_num)
3053{
3054 union sl2vl_tbl_to_u64 tmp_vltab;
3055 u8 vl;
3056
3057 if (sl > 15)
3058 return 0xf;
3059 tmp_vltab.sl64 = atomic64_read(&dev->sl2vl[port_num - 1]);
3060 vl = tmp_vltab.sl8[sl >> 1];
3061 if (sl & 1)
3062 vl &= 0x0f;
3063 else
3064 vl >>= 4;
3065 return vl;
3066}
3067
3068static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num,
3069 int index, union ib_gid *gid,
3070 enum ib_gid_type *gid_type)
3071{
3072 struct mlx4_ib_iboe *iboe = &ibdev->iboe;
3073 struct mlx4_port_gid_table *port_gid_table;
3074 unsigned long flags;
3075
3076 port_gid_table = &iboe->gids[port_num - 1];
3077 spin_lock_irqsave(&iboe->lock, flags);
3078 memcpy(gid, &port_gid_table->gids[index].gid, sizeof(*gid));
3079 *gid_type = port_gid_table->gids[index].gid_type;
3080 spin_unlock_irqrestore(&iboe->lock, flags);
3081 if (!memcmp(gid, &zgid, sizeof(*gid)))
3082 return -ENOENT;
3083
3084 return 0;
3085}
3086
3087#define MLX4_ROCEV2_QP1_SPORT 0xC000
3088static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
3089 void *wqe, unsigned *mlx_seg_len)
3090{
3091 struct ib_device *ib_dev = sqp->qp.ibqp.device;
3092 struct mlx4_ib_dev *ibdev = to_mdev(ib_dev);
3093 struct mlx4_wqe_mlx_seg *mlx = wqe;
3094 struct mlx4_wqe_ctrl_seg *ctrl = wqe;
3095 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
3096 struct mlx4_ib_ah *ah = to_mah(wr->ah);
3097 union ib_gid sgid;
3098 u16 pkey;
3099 int send_size;
3100 int header_size;
3101 int spc;
3102 int i;
3103 int err = 0;
3104 u16 vlan = 0xffff;
3105 bool is_eth;
3106 bool is_vlan = false;
3107 bool is_grh;
3108 bool is_udp = false;
3109 int ip_version = 0;
3110
3111 send_size = 0;
3112 for (i = 0; i < wr->wr.num_sge; ++i)
3113 send_size += wr->wr.sg_list[i].length;
3114
3115 is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
3116 is_grh = mlx4_ib_ah_grh_present(ah);
3117 if (is_eth) {
3118 enum ib_gid_type gid_type;
3119 if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3120
3121
3122
3123 err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev,
3124 be32_to_cpu(ah->av.ib.port_pd) >> 24,
3125 ah->av.ib.gid_index, &sgid.raw[0]);
3126 if (err)
3127 return err;
3128 } else {
3129 err = fill_gid_by_hw_index(ibdev, sqp->qp.port,
3130 ah->av.ib.gid_index,
3131 &sgid, &gid_type);
3132 if (!err) {
3133 is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
3134 if (is_udp) {
3135 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid))
3136 ip_version = 4;
3137 else
3138 ip_version = 6;
3139 is_grh = false;
3140 }
3141 } else {
3142 return err;
3143 }
3144 }
3145 if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
3146 vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
3147 is_vlan = 1;
3148 }
3149 }
3150 err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
3151 ip_version, is_udp, 0, &sqp->ud_header);
3152 if (err)
3153 return err;
3154
3155 if (!is_eth) {
3156 sqp->ud_header.lrh.service_level =
3157 be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
3158 sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
3159 sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
3160 }
3161
3162 if (is_grh || (ip_version == 6)) {
3163 sqp->ud_header.grh.traffic_class =
3164 (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3165 sqp->ud_header.grh.flow_label =
3166 ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
3167 sqp->ud_header.grh.hop_limit = ah->av.ib.hop_limit;
3168 if (is_eth) {
3169 memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16);
3170 } else {
3171 if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3172
3173
3174
3175
3176 sqp->ud_header.grh.source_gid.global.subnet_prefix =
3177 cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov.
3178 demux[sqp->qp.port - 1].
3179 subnet_prefix)));
3180 sqp->ud_header.grh.source_gid.global.interface_id =
3181 to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
3182 guid_cache[ah->av.ib.gid_index];
3183 } else {
3184 ib_get_cached_gid(ib_dev,
3185 be32_to_cpu(ah->av.ib.port_pd) >> 24,
3186 ah->av.ib.gid_index,
3187 &sqp->ud_header.grh.source_gid, NULL);
3188 }
3189 }
3190 memcpy(sqp->ud_header.grh.destination_gid.raw,
3191 ah->av.ib.dgid, 16);
3192 }
3193
3194 if (ip_version == 4) {
3195 sqp->ud_header.ip4.tos =
3196 (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3197 sqp->ud_header.ip4.id = 0;
3198 sqp->ud_header.ip4.frag_off = htons(IP_DF);
3199 sqp->ud_header.ip4.ttl = ah->av.eth.hop_limit;
3200
3201 memcpy(&sqp->ud_header.ip4.saddr,
3202 sgid.raw + 12, 4);
3203 memcpy(&sqp->ud_header.ip4.daddr, ah->av.ib.dgid + 12, 4);
3204 sqp->ud_header.ip4.check = ib_ud_ip4_csum(&sqp->ud_header);
3205 }
3206
3207 if (is_udp) {
3208 sqp->ud_header.udp.dport = htons(ROCE_V2_UDP_DPORT);
3209 sqp->ud_header.udp.sport = htons(MLX4_ROCEV2_QP1_SPORT);
3210 sqp->ud_header.udp.csum = 0;
3211 }
3212
3213 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
3214
3215 if (!is_eth) {
3216 mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
3217 (sqp->ud_header.lrh.destination_lid ==
3218 IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
3219 (sqp->ud_header.lrh.service_level << 8));
3220 if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
3221 mlx->flags |= cpu_to_be32(0x1);
3222 mlx->rlid = sqp->ud_header.lrh.destination_lid;
3223 }
3224
3225 switch (wr->wr.opcode) {
3226 case IB_WR_SEND:
3227 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
3228 sqp->ud_header.immediate_present = 0;
3229 break;
3230 case IB_WR_SEND_WITH_IMM:
3231 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
3232 sqp->ud_header.immediate_present = 1;
3233 sqp->ud_header.immediate_data = wr->wr.ex.imm_data;
3234 break;
3235 default:
3236 return -EINVAL;
3237 }
3238
3239 if (is_eth) {
3240 struct in6_addr in6;
3241 u16 ether_type;
3242 u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
3243
3244 ether_type = (!is_udp) ? ETH_P_IBOE:
3245 (ip_version == 4 ? ETH_P_IP : ETH_P_IPV6);
3246
3247 mlx->sched_prio = cpu_to_be16(pcp);
3248
3249 ether_addr_copy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac);
3250 memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
3251 memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2);
3252 memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4);
3253 memcpy(&in6, sgid.raw, sizeof(in6));
3254
3255
3256 if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
3257 mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
3258 if (!is_vlan) {
3259 sqp->ud_header.eth.type = cpu_to_be16(ether_type);
3260 } else {
3261 sqp->ud_header.vlan.type = cpu_to_be16(ether_type);
3262 sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
3263 }
3264 } else {
3265 sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 :
3266 sl_to_vl(to_mdev(ib_dev),
3267 sqp->ud_header.lrh.service_level,
3268 sqp->qp.port);
3269 if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
3270 return -EINVAL;
3271 if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
3272 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
3273 }
3274 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
3275 if (!sqp->qp.ibqp.qp_num)
3276 ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
3277 else
3278 ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index, &pkey);
3279 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
3280 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
3281 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
3282 sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ?
3283 sqp->qkey : wr->remote_qkey);
3284 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
3285
3286 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3287
3288 if (0) {
3289 pr_err("built UD header of size %d:\n", header_size);
3290 for (i = 0; i < header_size / 4; ++i) {
3291 if (i % 8 == 0)
3292 pr_err(" [%02x] ", i * 4);
3293 pr_cont(" %08x",
3294 be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
3295 if ((i + 1) % 8 == 0)
3296 pr_cont("\n");
3297 }
3298 pr_err("\n");
3299 }
3300
3301
3302
3303
3304
3305
3306
3307 spc = MLX4_INLINE_ALIGN -
3308 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3309 if (header_size <= spc) {
3310 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
3311 memcpy(inl + 1, sqp->header_buf, header_size);
3312 i = 1;
3313 } else {
3314 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3315 memcpy(inl + 1, sqp->header_buf, spc);
3316
3317 inl = (void *) (inl + 1) + spc;
3318 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332 wmb();
3333 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
3334 i = 2;
3335 }
3336
3337 *mlx_seg_len =
3338 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
3339 return 0;
3340}
3341
3342static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
3343{
3344 unsigned cur;
3345 struct mlx4_ib_cq *cq;
3346
3347 cur = wq->head - wq->tail;
3348 if (likely(cur + nreq < wq->max_post))
3349 return 0;
3350
3351 cq = to_mcq(ib_cq);
3352 spin_lock(&cq->lock);
3353 cur = wq->head - wq->tail;
3354 spin_unlock(&cq->lock);
3355
3356 return cur + nreq >= wq->max_post;
3357}
3358
3359static __be32 convert_access(int acc)
3360{
3361 return (acc & IB_ACCESS_REMOTE_ATOMIC ?
3362 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC) : 0) |
3363 (acc & IB_ACCESS_REMOTE_WRITE ?
3364 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) |
3365 (acc & IB_ACCESS_REMOTE_READ ?
3366 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ) : 0) |
3367 (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) |
3368 cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
3369}
3370
3371static void set_reg_seg(struct mlx4_wqe_fmr_seg *fseg,
3372 struct ib_reg_wr *wr)
3373{
3374 struct mlx4_ib_mr *mr = to_mmr(wr->mr);
3375
3376 fseg->flags = convert_access(wr->access);
3377 fseg->mem_key = cpu_to_be32(wr->key);
3378 fseg->buf_list = cpu_to_be64(mr->page_map);
3379 fseg->start_addr = cpu_to_be64(mr->ibmr.iova);
3380 fseg->reg_len = cpu_to_be64(mr->ibmr.length);
3381 fseg->offset = 0;
3382 fseg->page_size = cpu_to_be32(ilog2(mr->ibmr.page_size));
3383 fseg->reserved[0] = 0;
3384 fseg->reserved[1] = 0;
3385}
3386
3387static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
3388{
3389 memset(iseg, 0, sizeof(*iseg));
3390 iseg->mem_key = cpu_to_be32(rkey);
3391}
3392
3393static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
3394 u64 remote_addr, u32 rkey)
3395{
3396 rseg->raddr = cpu_to_be64(remote_addr);
3397 rseg->rkey = cpu_to_be32(rkey);
3398 rseg->reserved = 0;
3399}
3400
3401static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg,
3402 struct ib_atomic_wr *wr)
3403{
3404 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
3405 aseg->swap_add = cpu_to_be64(wr->swap);
3406 aseg->compare = cpu_to_be64(wr->compare_add);
3407 } else if (wr->wr.opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
3408 aseg->swap_add = cpu_to_be64(wr->compare_add);
3409 aseg->compare = cpu_to_be64(wr->compare_add_mask);
3410 } else {
3411 aseg->swap_add = cpu_to_be64(wr->compare_add);
3412 aseg->compare = 0;
3413 }
3414
3415}
3416
3417static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
3418 struct ib_atomic_wr *wr)
3419{
3420 aseg->swap_add = cpu_to_be64(wr->swap);
3421 aseg->swap_add_mask = cpu_to_be64(wr->swap_mask);
3422 aseg->compare = cpu_to_be64(wr->compare_add);
3423 aseg->compare_mask = cpu_to_be64(wr->compare_add_mask);
3424}
3425
3426static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
3427 struct ib_ud_wr *wr)
3428{
3429 memcpy(dseg->av, &to_mah(wr->ah)->av, sizeof (struct mlx4_av));
3430 dseg->dqpn = cpu_to_be32(wr->remote_qpn);
3431 dseg->qkey = cpu_to_be32(wr->remote_qkey);
3432 dseg->vlan = to_mah(wr->ah)->av.eth.vlan;
3433 memcpy(dseg->mac, to_mah(wr->ah)->av.eth.mac, 6);
3434}
3435
3436static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev,
3437 struct mlx4_wqe_datagram_seg *dseg,
3438 struct ib_ud_wr *wr,
3439 enum mlx4_ib_qp_type qpt)
3440{
3441 union mlx4_ext_av *av = &to_mah(wr->ah)->av;
3442 struct mlx4_av sqp_av = {0};
3443 int port = *((u8 *) &av->ib.port_pd) & 0x3;
3444
3445
3446 sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000);
3447 sqp_av.g_slid = av->ib.g_slid & 0x7f;
3448 sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel &
3449 cpu_to_be32(0xf0000000);
3450
3451 memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av));
3452 if (qpt == MLX4_IB_QPT_PROXY_GSI)
3453 dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp1_tunnel);
3454 else
3455 dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp0_tunnel);
3456
3457 dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY);
3458}
3459
3460static void build_tunnel_header(struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len)
3461{
3462 struct mlx4_wqe_inline_seg *inl = wqe;
3463 struct mlx4_ib_tunnel_header hdr;
3464 struct mlx4_ib_ah *ah = to_mah(wr->ah);
3465 int spc;
3466 int i;
3467
3468 memcpy(&hdr.av, &ah->av, sizeof hdr.av);
3469 hdr.remote_qpn = cpu_to_be32(wr->remote_qpn);
3470 hdr.pkey_index = cpu_to_be16(wr->pkey_index);
3471 hdr.qkey = cpu_to_be32(wr->remote_qkey);
3472 memcpy(hdr.mac, ah->av.eth.mac, 6);
3473 hdr.vlan = ah->av.eth.vlan;
3474
3475 spc = MLX4_INLINE_ALIGN -
3476 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3477 if (sizeof (hdr) <= spc) {
3478 memcpy(inl + 1, &hdr, sizeof (hdr));
3479 wmb();
3480 inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr));
3481 i = 1;
3482 } else {
3483 memcpy(inl + 1, &hdr, spc);
3484 wmb();
3485 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3486
3487 inl = (void *) (inl + 1) + spc;
3488 memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc);
3489 wmb();
3490 inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc));
3491 i = 2;
3492 }
3493
3494 *mlx_seg_len =
3495 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16);
3496}
3497
3498static void set_mlx_icrc_seg(void *dseg)
3499{
3500 u32 *t = dseg;
3501 struct mlx4_wqe_inline_seg *iseg = dseg;
3502
3503 t[1] = 0;
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513 wmb();
3514
3515 iseg->byte_count = cpu_to_be32((1 << 31) | 4);
3516}
3517
3518static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3519{
3520 dseg->lkey = cpu_to_be32(sg->lkey);
3521 dseg->addr = cpu_to_be64(sg->addr);
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531 wmb();
3532
3533 dseg->byte_count = cpu_to_be32(sg->length);
3534}
3535
3536static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3537{
3538 dseg->byte_count = cpu_to_be32(sg->length);
3539 dseg->lkey = cpu_to_be32(sg->lkey);
3540 dseg->addr = cpu_to_be64(sg->addr);
3541}
3542
3543static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_ud_wr *wr,
3544 struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
3545 __be32 *lso_hdr_sz, __be32 *blh)
3546{
3547 unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16);
3548
3549 if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
3550 *blh = cpu_to_be32(1 << 6);
3551
3552 if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
3553 wr->wr.num_sge > qp->sq.max_gs - (halign >> 4)))
3554 return -EINVAL;
3555
3556 memcpy(wqe->header, wr->header, wr->hlen);
3557
3558 *lso_hdr_sz = cpu_to_be32(wr->mss << 16 | wr->hlen);
3559 *lso_seg_len = halign;
3560 return 0;
3561}
3562
3563static __be32 send_ieth(struct ib_send_wr *wr)
3564{
3565 switch (wr->opcode) {
3566 case IB_WR_SEND_WITH_IMM:
3567 case IB_WR_RDMA_WRITE_WITH_IMM:
3568 return wr->ex.imm_data;
3569
3570 case IB_WR_SEND_WITH_INV:
3571 return cpu_to_be32(wr->ex.invalidate_rkey);
3572
3573 default:
3574 return 0;
3575 }
3576}
3577
3578static void add_zero_len_inline(void *wqe)
3579{
3580 struct mlx4_wqe_inline_seg *inl = wqe;
3581 memset(wqe, 0, 16);
3582 inl->byte_count = cpu_to_be32(1 << 31);
3583}
3584
3585int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3586 struct ib_send_wr **bad_wr)
3587{
3588 struct mlx4_ib_qp *qp = to_mqp(ibqp);
3589 void *wqe;
3590 struct mlx4_wqe_ctrl_seg *ctrl;
3591 struct mlx4_wqe_data_seg *dseg;
3592 unsigned long flags;
3593 int nreq;
3594 int err = 0;
3595 unsigned ind;
3596 int uninitialized_var(stamp);
3597 int uninitialized_var(size);
3598 unsigned uninitialized_var(seglen);
3599 __be32 dummy;
3600 __be32 *lso_wqe;
3601 __be32 uninitialized_var(lso_hdr_sz);
3602 __be32 blh;
3603 int i;
3604 struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3605
3606 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
3607 struct mlx4_ib_sqp *sqp = to_msqp(qp);
3608
3609 if (sqp->roce_v2_gsi) {
3610 struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
3611 enum ib_gid_type gid_type;
3612 union ib_gid gid;
3613
3614 if (!fill_gid_by_hw_index(mdev, sqp->qp.port,
3615 ah->av.ib.gid_index,
3616 &gid, &gid_type))
3617 qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
3618 to_mqp(sqp->roce_v2_gsi) : qp;
3619 else
3620 pr_err("Failed to get gid at index %d. RoCEv2 will not work properly\n",
3621 ah->av.ib.gid_index);
3622 }
3623 }
3624
3625 spin_lock_irqsave(&qp->sq.lock, flags);
3626 if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
3627 err = -EIO;
3628 *bad_wr = wr;
3629 nreq = 0;
3630 goto out;
3631 }
3632
3633 ind = qp->sq_next_wqe;
3634
3635 for (nreq = 0; wr; ++nreq, wr = wr->next) {
3636 lso_wqe = &dummy;
3637 blh = 0;
3638
3639 if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
3640 err = -ENOMEM;
3641 *bad_wr = wr;
3642 goto out;
3643 }
3644
3645 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
3646 err = -EINVAL;
3647 *bad_wr = wr;
3648 goto out;
3649 }
3650
3651 ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
3652 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
3653
3654 ctrl->srcrb_flags =
3655 (wr->send_flags & IB_SEND_SIGNALED ?
3656 cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
3657 (wr->send_flags & IB_SEND_SOLICITED ?
3658 cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
3659 ((wr->send_flags & IB_SEND_IP_CSUM) ?
3660 cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
3661 MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
3662 qp->sq_signal_bits;
3663
3664 ctrl->imm = send_ieth(wr);
3665
3666 wqe += sizeof *ctrl;
3667 size = sizeof *ctrl / 16;
3668
3669 switch (qp->mlx4_ib_qp_type) {
3670 case MLX4_IB_QPT_RC:
3671 case MLX4_IB_QPT_UC:
3672 switch (wr->opcode) {
3673 case IB_WR_ATOMIC_CMP_AND_SWP:
3674 case IB_WR_ATOMIC_FETCH_AND_ADD:
3675 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
3676 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3677 atomic_wr(wr)->rkey);
3678 wqe += sizeof (struct mlx4_wqe_raddr_seg);
3679
3680 set_atomic_seg(wqe, atomic_wr(wr));
3681 wqe += sizeof (struct mlx4_wqe_atomic_seg);
3682
3683 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3684 sizeof (struct mlx4_wqe_atomic_seg)) / 16;
3685
3686 break;
3687
3688 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
3689 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3690 atomic_wr(wr)->rkey);
3691 wqe += sizeof (struct mlx4_wqe_raddr_seg);
3692
3693 set_masked_atomic_seg(wqe, atomic_wr(wr));
3694 wqe += sizeof (struct mlx4_wqe_masked_atomic_seg);
3695
3696 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3697 sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
3698
3699 break;
3700
3701 case IB_WR_RDMA_READ:
3702 case IB_WR_RDMA_WRITE:
3703 case IB_WR_RDMA_WRITE_WITH_IMM:
3704 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,
3705 rdma_wr(wr)->rkey);
3706 wqe += sizeof (struct mlx4_wqe_raddr_seg);
3707 size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
3708 break;
3709
3710 case IB_WR_LOCAL_INV:
3711 ctrl->srcrb_flags |=
3712 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3713 set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
3714 wqe += sizeof (struct mlx4_wqe_local_inval_seg);
3715 size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
3716 break;
3717
3718 case IB_WR_REG_MR:
3719 ctrl->srcrb_flags |=
3720 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3721 set_reg_seg(wqe, reg_wr(wr));
3722 wqe += sizeof(struct mlx4_wqe_fmr_seg);
3723 size += sizeof(struct mlx4_wqe_fmr_seg) / 16;
3724 break;
3725
3726 default:
3727
3728 break;
3729 }
3730 break;
3731
3732 case MLX4_IB_QPT_TUN_SMI_OWNER:
3733 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3734 ctrl, &seglen);
3735 if (unlikely(err)) {
3736 *bad_wr = wr;
3737 goto out;
3738 }
3739 wqe += seglen;
3740 size += seglen / 16;
3741 break;
3742 case MLX4_IB_QPT_TUN_SMI:
3743 case MLX4_IB_QPT_TUN_GSI:
3744
3745 set_datagram_seg(wqe, ud_wr(wr));
3746
3747 *(__be32 *) wqe |= cpu_to_be32(0x80000000);
3748 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3749 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3750 break;
3751 case MLX4_IB_QPT_UD:
3752 set_datagram_seg(wqe, ud_wr(wr));
3753 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3754 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3755
3756 if (wr->opcode == IB_WR_LSO) {
3757 err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen,
3758 &lso_hdr_sz, &blh);
3759 if (unlikely(err)) {
3760 *bad_wr = wr;
3761 goto out;
3762 }
3763 lso_wqe = (__be32 *) wqe;
3764 wqe += seglen;
3765 size += seglen / 16;
3766 }
3767 break;
3768
3769 case MLX4_IB_QPT_PROXY_SMI_OWNER:
3770 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3771 ctrl, &seglen);
3772 if (unlikely(err)) {
3773 *bad_wr = wr;
3774 goto out;
3775 }
3776 wqe += seglen;
3777 size += seglen / 16;
3778
3779 add_zero_len_inline(wqe);
3780 wqe += 16;
3781 size++;
3782 build_tunnel_header(ud_wr(wr), wqe, &seglen);
3783 wqe += seglen;
3784 size += seglen / 16;
3785 break;
3786 case MLX4_IB_QPT_PROXY_SMI:
3787 case MLX4_IB_QPT_PROXY_GSI:
3788
3789
3790
3791
3792 set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe,
3793 ud_wr(wr),
3794 qp->mlx4_ib_qp_type);
3795 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3796 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3797 build_tunnel_header(ud_wr(wr), wqe, &seglen);
3798 wqe += seglen;
3799 size += seglen / 16;
3800 break;
3801
3802 case MLX4_IB_QPT_SMI:
3803 case MLX4_IB_QPT_GSI:
3804 err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl,
3805 &seglen);
3806 if (unlikely(err)) {
3807 *bad_wr = wr;
3808 goto out;
3809 }
3810 wqe += seglen;
3811 size += seglen / 16;
3812 break;
3813
3814 default:
3815 break;
3816 }
3817
3818
3819
3820
3821
3822
3823
3824
3825 dseg = wqe;
3826 dseg += wr->num_sge - 1;
3827 size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
3828
3829
3830 if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
3831 qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI ||
3832 qp->mlx4_ib_qp_type &
3833 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) {
3834 set_mlx_icrc_seg(dseg + 1);
3835 size += sizeof (struct mlx4_wqe_data_seg) / 16;
3836 }
3837
3838 for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
3839 set_data_seg(dseg, wr->sg_list + i);
3840
3841
3842
3843
3844
3845
3846 wmb();
3847 *lso_wqe = lso_hdr_sz;
3848
3849 ctrl->qpn_vlan.fence_size = (wr->send_flags & IB_SEND_FENCE ?
3850 MLX4_WQE_CTRL_FENCE : 0) | size;
3851
3852
3853
3854
3855
3856
3857 wmb();
3858
3859 if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
3860 *bad_wr = wr;
3861 err = -EINVAL;
3862 goto out;
3863 }
3864
3865 ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
3866 (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
3867
3868 stamp = ind + qp->sq_spare_wqes;
3869 ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880 if (wr->next) {
3881 stamp_send_wqe(qp, stamp, size * 16);
3882 ind = pad_wraparound(qp, ind);
3883 }
3884 }
3885
3886out:
3887 if (likely(nreq)) {
3888 qp->sq.head += nreq;
3889
3890
3891
3892
3893
3894 wmb();
3895
3896 writel_relaxed(qp->doorbell_qpn,
3897 to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
3898
3899
3900
3901
3902
3903 mmiowb();
3904
3905 stamp_send_wqe(qp, stamp, size * 16);
3906
3907 ind = pad_wraparound(qp, ind);
3908 qp->sq_next_wqe = ind;
3909 }
3910
3911 spin_unlock_irqrestore(&qp->sq.lock, flags);
3912
3913 return err;
3914}
3915
3916int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3917 struct ib_recv_wr **bad_wr)
3918{
3919 struct mlx4_ib_qp *qp = to_mqp(ibqp);
3920 struct mlx4_wqe_data_seg *scat;
3921 unsigned long flags;
3922 int err = 0;
3923 int nreq;
3924 int ind;
3925 int max_gs;
3926 int i;
3927 struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3928
3929 max_gs = qp->rq.max_gs;
3930 spin_lock_irqsave(&qp->rq.lock, flags);
3931
3932 if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
3933 err = -EIO;
3934 *bad_wr = wr;
3935 nreq = 0;
3936 goto out;
3937 }
3938
3939 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
3940
3941 for (nreq = 0; wr; ++nreq, wr = wr->next) {
3942 if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
3943 err = -ENOMEM;
3944 *bad_wr = wr;
3945 goto out;
3946 }
3947
3948 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
3949 err = -EINVAL;
3950 *bad_wr = wr;
3951 goto out;
3952 }
3953
3954 scat = get_recv_wqe(qp, ind);
3955
3956 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
3957 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
3958 ib_dma_sync_single_for_device(ibqp->device,
3959 qp->sqp_proxy_rcv[ind].map,
3960 sizeof (struct mlx4_ib_proxy_sqp_hdr),
3961 DMA_FROM_DEVICE);
3962 scat->byte_count =
3963 cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr));
3964
3965 scat->lkey = cpu_to_be32(wr->sg_list->lkey);
3966 scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map);
3967 scat++;
3968 max_gs--;
3969 }
3970
3971 for (i = 0; i < wr->num_sge; ++i)
3972 __set_data_seg(scat + i, wr->sg_list + i);
3973
3974 if (i < max_gs) {
3975 scat[i].byte_count = 0;
3976 scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
3977 scat[i].addr = 0;
3978 }
3979
3980 qp->rq.wrid[ind] = wr->wr_id;
3981
3982 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
3983 }
3984
3985out:
3986 if (likely(nreq)) {
3987 qp->rq.head += nreq;
3988
3989
3990
3991
3992
3993 wmb();
3994
3995 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
3996 }
3997
3998 spin_unlock_irqrestore(&qp->rq.lock, flags);
3999
4000 return err;
4001}
4002
4003static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
4004{
4005 switch (mlx4_state) {
4006 case MLX4_QP_STATE_RST: return IB_QPS_RESET;
4007 case MLX4_QP_STATE_INIT: return IB_QPS_INIT;
4008 case MLX4_QP_STATE_RTR: return IB_QPS_RTR;
4009 case MLX4_QP_STATE_RTS: return IB_QPS_RTS;
4010 case MLX4_QP_STATE_SQ_DRAINING:
4011 case MLX4_QP_STATE_SQD: return IB_QPS_SQD;
4012 case MLX4_QP_STATE_SQER: return IB_QPS_SQE;
4013 case MLX4_QP_STATE_ERR: return IB_QPS_ERR;
4014 default: return -1;
4015 }
4016}
4017
4018static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
4019{
4020 switch (mlx4_mig_state) {
4021 case MLX4_QP_PM_ARMED: return IB_MIG_ARMED;
4022 case MLX4_QP_PM_REARM: return IB_MIG_REARM;
4023 case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
4024 default: return -1;
4025 }
4026}
4027
4028static int to_ib_qp_access_flags(int mlx4_flags)
4029{
4030 int ib_flags = 0;
4031
4032 if (mlx4_flags & MLX4_QP_BIT_RRE)
4033 ib_flags |= IB_ACCESS_REMOTE_READ;
4034 if (mlx4_flags & MLX4_QP_BIT_RWE)
4035 ib_flags |= IB_ACCESS_REMOTE_WRITE;
4036 if (mlx4_flags & MLX4_QP_BIT_RAE)
4037 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
4038
4039 return ib_flags;
4040}
4041
4042static void to_rdma_ah_attr(struct mlx4_ib_dev *ibdev,
4043 struct rdma_ah_attr *ah_attr,
4044 struct mlx4_qp_path *path)
4045{
4046 struct mlx4_dev *dev = ibdev->dev;
4047 u8 port_num = path->sched_queue & 0x40 ? 2 : 1;
4048
4049 memset(ah_attr, 0, sizeof(*ah_attr));
4050 ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port_num);
4051 if (port_num == 0 || port_num > dev->caps.num_ports)
4052 return;
4053
4054 if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE)
4055 rdma_ah_set_sl(ah_attr, ((path->sched_queue >> 3) & 0x7) |
4056 ((path->sched_queue & 4) << 1));
4057 else
4058 rdma_ah_set_sl(ah_attr, (path->sched_queue >> 2) & 0xf);
4059 rdma_ah_set_port_num(ah_attr, port_num);
4060
4061 rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
4062 rdma_ah_set_path_bits(ah_attr, path->grh_mylmc & 0x7f);
4063 rdma_ah_set_static_rate(ah_attr,
4064 path->static_rate ? path->static_rate - 5 : 0);
4065 if (path->grh_mylmc & (1 << 7)) {
4066 rdma_ah_set_grh(ah_attr, NULL,
4067 be32_to_cpu(path->tclass_flowlabel) & 0xfffff,
4068 path->mgid_index,
4069 path->hop_limit,
4070 (be32_to_cpu(path->tclass_flowlabel)
4071 >> 20) & 0xff);
4072 rdma_ah_set_dgid_raw(ah_attr, path->rgid);
4073 }
4074}
4075
4076int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
4077 struct ib_qp_init_attr *qp_init_attr)
4078{
4079 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
4080 struct mlx4_ib_qp *qp = to_mqp(ibqp);
4081 struct mlx4_qp_context context;
4082 int mlx4_state;
4083 int err = 0;
4084
4085 if (ibqp->rwq_ind_tbl)
4086 return -EOPNOTSUPP;
4087
4088 mutex_lock(&qp->mutex);
4089
4090 if (qp->state == IB_QPS_RESET) {
4091 qp_attr->qp_state = IB_QPS_RESET;
4092 goto done;
4093 }
4094
4095 err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
4096 if (err) {
4097 err = -EINVAL;
4098 goto out;
4099 }
4100
4101 mlx4_state = be32_to_cpu(context.flags) >> 28;
4102
4103 qp->state = to_ib_qp_state(mlx4_state);
4104 qp_attr->qp_state = qp->state;
4105 qp_attr->path_mtu = context.mtu_msgmax >> 5;
4106 qp_attr->path_mig_state =
4107 to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
4108 qp_attr->qkey = be32_to_cpu(context.qkey);
4109 qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
4110 qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff;
4111 qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff;
4112 qp_attr->qp_access_flags =
4113 to_ib_qp_access_flags(be32_to_cpu(context.params2));
4114
4115 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
4116 to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
4117 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
4118 qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
4119 qp_attr->alt_port_num =
4120 rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
4121 }
4122
4123 qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
4124 if (qp_attr->qp_state == IB_QPS_INIT)
4125 qp_attr->port_num = qp->port;
4126 else
4127 qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
4128
4129
4130 qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
4131
4132 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
4133
4134 qp_attr->max_dest_rd_atomic =
4135 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
4136 qp_attr->min_rnr_timer =
4137 (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
4138 qp_attr->timeout = context.pri_path.ackto >> 3;
4139 qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7;
4140 qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7;
4141 qp_attr->alt_timeout = context.alt_path.ackto >> 3;
4142
4143done:
4144 qp_attr->cur_qp_state = qp_attr->qp_state;
4145 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
4146 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
4147
4148 if (!ibqp->uobject) {
4149 qp_attr->cap.max_send_wr = qp->sq.wqe_cnt;
4150 qp_attr->cap.max_send_sge = qp->sq.max_gs;
4151 } else {
4152 qp_attr->cap.max_send_wr = 0;
4153 qp_attr->cap.max_send_sge = 0;
4154 }
4155
4156
4157
4158
4159
4160 qp_attr->cap.max_inline_data = 0;
4161
4162 qp_init_attr->cap = qp_attr->cap;
4163
4164 qp_init_attr->create_flags = 0;
4165 if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
4166 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
4167
4168 if (qp->flags & MLX4_IB_QP_LSO)
4169 qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
4170
4171 if (qp->flags & MLX4_IB_QP_NETIF)
4172 qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP;
4173
4174 qp_init_attr->sq_sig_type =
4175 qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ?
4176 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
4177
4178out:
4179 mutex_unlock(&qp->mutex);
4180 return err;
4181}
4182
4183struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
4184 struct ib_wq_init_attr *init_attr,
4185 struct ib_udata *udata)
4186{
4187 struct mlx4_ib_dev *dev;
4188 struct ib_qp_init_attr ib_qp_init_attr;
4189 struct mlx4_ib_qp *qp;
4190 struct mlx4_ib_create_wq ucmd;
4191 int err, required_cmd_sz;
4192
4193 if (!(udata && pd->uobject))
4194 return ERR_PTR(-EINVAL);
4195
4196 required_cmd_sz = offsetof(typeof(ucmd), comp_mask) +
4197 sizeof(ucmd.comp_mask);
4198 if (udata->inlen < required_cmd_sz) {
4199 pr_debug("invalid inlen\n");
4200 return ERR_PTR(-EINVAL);
4201 }
4202
4203 if (udata->inlen > sizeof(ucmd) &&
4204 !ib_is_udata_cleared(udata, sizeof(ucmd),
4205 udata->inlen - sizeof(ucmd))) {
4206 pr_debug("inlen is not supported\n");
4207 return ERR_PTR(-EOPNOTSUPP);
4208 }
4209
4210 if (udata->outlen)
4211 return ERR_PTR(-EOPNOTSUPP);
4212
4213 dev = to_mdev(pd->device);
4214
4215 if (init_attr->wq_type != IB_WQT_RQ) {
4216 pr_debug("unsupported wq type %d\n", init_attr->wq_type);
4217 return ERR_PTR(-EOPNOTSUPP);
4218 }
4219
4220 if (init_attr->create_flags & ~IB_WQ_FLAGS_SCATTER_FCS) {
4221 pr_debug("unsupported create_flags %u\n",
4222 init_attr->create_flags);
4223 return ERR_PTR(-EOPNOTSUPP);
4224 }
4225
4226 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
4227 if (!qp)
4228 return ERR_PTR(-ENOMEM);
4229
4230 qp->pri.vid = 0xFFFF;
4231 qp->alt.vid = 0xFFFF;
4232
4233 memset(&ib_qp_init_attr, 0, sizeof(ib_qp_init_attr));
4234 ib_qp_init_attr.qp_context = init_attr->wq_context;
4235 ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;
4236 ib_qp_init_attr.cap.max_recv_wr = init_attr->max_wr;
4237 ib_qp_init_attr.cap.max_recv_sge = init_attr->max_sge;
4238 ib_qp_init_attr.recv_cq = init_attr->cq;
4239 ib_qp_init_attr.send_cq = ib_qp_init_attr.recv_cq;
4240
4241 if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS)
4242 ib_qp_init_attr.create_flags |= IB_QP_CREATE_SCATTER_FCS;
4243
4244 err = create_qp_common(dev, pd, MLX4_IB_RWQ_SRC, &ib_qp_init_attr,
4245 udata, 0, &qp);
4246 if (err) {
4247 kfree(qp);
4248 return ERR_PTR(err);
4249 }
4250
4251 qp->ibwq.event_handler = init_attr->event_handler;
4252 qp->ibwq.wq_num = qp->mqp.qpn;
4253 qp->ibwq.state = IB_WQS_RESET;
4254
4255 return &qp->ibwq;
4256}
4257
4258static int ib_wq2qp_state(enum ib_wq_state state)
4259{
4260 switch (state) {
4261 case IB_WQS_RESET:
4262 return IB_QPS_RESET;
4263 case IB_WQS_RDY:
4264 return IB_QPS_RTR;
4265 default:
4266 return IB_QPS_ERR;
4267 }
4268}
4269
4270static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state)
4271{
4272 struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4273 enum ib_qp_state qp_cur_state;
4274 enum ib_qp_state qp_new_state;
4275 int attr_mask;
4276 int err;
4277
4278
4279
4280
4281 qp_cur_state = qp->state;
4282 qp_new_state = ib_wq2qp_state(new_state);
4283
4284 if (ib_wq2qp_state(new_state) == qp_cur_state)
4285 return 0;
4286
4287 if (new_state == IB_WQS_RDY) {
4288 struct ib_qp_attr attr = {};
4289
4290 attr.port_num = qp->port;
4291 attr_mask = IB_QP_PORT;
4292
4293 err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, &attr,
4294 attr_mask, IB_QPS_RESET, IB_QPS_INIT);
4295 if (err) {
4296 pr_debug("WQN=0x%06x failed to apply RST->INIT on the HW QP\n",
4297 ibwq->wq_num);
4298 return err;
4299 }
4300
4301 qp_cur_state = IB_QPS_INIT;
4302 }
4303
4304 attr_mask = 0;
4305 err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL, attr_mask,
4306 qp_cur_state, qp_new_state);
4307
4308 if (err && (qp_cur_state == IB_QPS_INIT)) {
4309 qp_new_state = IB_QPS_RESET;
4310 if (__mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL,
4311 attr_mask, IB_QPS_INIT, IB_QPS_RESET)) {
4312 pr_warn("WQN=0x%06x failed with reverting HW's resources failure\n",
4313 ibwq->wq_num);
4314 qp_new_state = IB_QPS_INIT;
4315 }
4316 }
4317
4318 qp->state = qp_new_state;
4319
4320 return err;
4321}
4322
4323int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,
4324 u32 wq_attr_mask, struct ib_udata *udata)
4325{
4326 struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4327 struct mlx4_ib_modify_wq ucmd = {};
4328 size_t required_cmd_sz;
4329 enum ib_wq_state cur_state, new_state;
4330 int err = 0;
4331
4332 required_cmd_sz = offsetof(typeof(ucmd), reserved) +
4333 sizeof(ucmd.reserved);
4334 if (udata->inlen < required_cmd_sz)
4335 return -EINVAL;
4336
4337 if (udata->inlen > sizeof(ucmd) &&
4338 !ib_is_udata_cleared(udata, sizeof(ucmd),
4339 udata->inlen - sizeof(ucmd)))
4340 return -EOPNOTSUPP;
4341
4342 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
4343 return -EFAULT;
4344
4345 if (ucmd.comp_mask || ucmd.reserved)
4346 return -EOPNOTSUPP;
4347
4348 if (wq_attr_mask & IB_WQ_FLAGS)
4349 return -EOPNOTSUPP;
4350
4351 cur_state = wq_attr_mask & IB_WQ_CUR_STATE ? wq_attr->curr_wq_state :
4352 ibwq->state;
4353 new_state = wq_attr_mask & IB_WQ_STATE ? wq_attr->wq_state : cur_state;
4354
4355 if (cur_state < IB_WQS_RESET || cur_state > IB_WQS_ERR ||
4356 new_state < IB_WQS_RESET || new_state > IB_WQS_ERR)
4357 return -EINVAL;
4358
4359 if ((new_state == IB_WQS_RDY) && (cur_state == IB_WQS_ERR))
4360 return -EINVAL;
4361
4362 if ((new_state == IB_WQS_ERR) && (cur_state == IB_WQS_RESET))
4363 return -EINVAL;
4364
4365
4366
4367
4368 mutex_lock(&qp->mutex);
4369
4370
4371
4372
4373 if (qp->rss_usecnt)
4374 err = _mlx4_ib_modify_wq(ibwq, new_state);
4375
4376 if (!err)
4377 ibwq->state = new_state;
4378
4379 mutex_unlock(&qp->mutex);
4380
4381 return err;
4382}
4383
4384int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
4385{
4386 struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
4387 struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4388
4389 if (qp->counter_index)
4390 mlx4_ib_free_qp_counter(dev, qp);
4391
4392 destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, 1);
4393
4394 kfree(qp);
4395
4396 return 0;
4397}
4398
4399struct ib_rwq_ind_table
4400*mlx4_ib_create_rwq_ind_table(struct ib_device *device,
4401 struct ib_rwq_ind_table_init_attr *init_attr,
4402 struct ib_udata *udata)
4403{
4404 struct ib_rwq_ind_table *rwq_ind_table;
4405 struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};
4406 unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size;
4407 unsigned int base_wqn;
4408 size_t min_resp_len;
4409 int i;
4410 int err;
4411
4412 if (udata->inlen > 0 &&
4413 !ib_is_udata_cleared(udata, 0,
4414 udata->inlen))
4415 return ERR_PTR(-EOPNOTSUPP);
4416
4417 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4418 if (udata->outlen && udata->outlen < min_resp_len)
4419 return ERR_PTR(-EINVAL);
4420
4421 if (ind_tbl_size >
4422 device->attrs.rss_caps.max_rwq_indirection_table_size) {
4423 pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",
4424 ind_tbl_size,
4425 device->attrs.rss_caps.max_rwq_indirection_table_size);
4426 return ERR_PTR(-EINVAL);
4427 }
4428
4429 base_wqn = init_attr->ind_tbl[0]->wq_num;
4430
4431 if (base_wqn % ind_tbl_size) {
4432 pr_debug("WQN=0x%x isn't aligned with indirection table size\n",
4433 base_wqn);
4434 return ERR_PTR(-EINVAL);
4435 }
4436
4437 for (i = 1; i < ind_tbl_size; i++) {
4438 if (++base_wqn != init_attr->ind_tbl[i]->wq_num) {
4439 pr_debug("indirection table's WQNs aren't consecutive\n");
4440 return ERR_PTR(-EINVAL);
4441 }
4442 }
4443
4444 rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL);
4445 if (!rwq_ind_table)
4446 return ERR_PTR(-ENOMEM);
4447
4448 if (udata->outlen) {
4449 resp.response_length = offsetof(typeof(resp), response_length) +
4450 sizeof(resp.response_length);
4451 err = ib_copy_to_udata(udata, &resp, resp.response_length);
4452 if (err)
4453 goto err;
4454 }
4455
4456 return rwq_ind_table;
4457
4458err:
4459 kfree(rwq_ind_table);
4460 return ERR_PTR(err);
4461}
4462
4463int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
4464{
4465 kfree(ib_rwq_ind_tbl);
4466 return 0;
4467}
4468