1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/log2.h>
35#include <linux/etherdevice.h>
36#include <net/ip.h>
37#include <linux/slab.h>
38#include <linux/netdevice.h>
39
40#include <rdma/ib_cache.h>
41#include <rdma/ib_pack.h>
42#include <rdma/ib_addr.h>
43#include <rdma/ib_mad.h>
44
45#include <linux/mlx4/driver.h>
46#include <linux/mlx4/qp.h>
47
48#include "mlx4_ib.h"
49#include <rdma/mlx4-abi.h>
50
51static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
52 struct mlx4_ib_cq *recv_cq);
53static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
54 struct mlx4_ib_cq *recv_cq);
55static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state);
56
57enum {
58 MLX4_IB_ACK_REQ_FREQ = 8,
59};
60
61enum {
62 MLX4_IB_DEFAULT_SCHED_QUEUE = 0x83,
63 MLX4_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
64 MLX4_IB_LINK_TYPE_IB = 0,
65 MLX4_IB_LINK_TYPE_ETH = 1
66};
67
68enum {
69
70
71
72
73
74
75 MLX4_IB_UD_HEADER_SIZE = 82,
76 MLX4_IB_LSO_HEADER_SPARE = 128,
77};
78
79struct mlx4_ib_sqp {
80 struct mlx4_ib_qp qp;
81 int pkey_index;
82 u32 qkey;
83 u32 send_psn;
84 struct ib_ud_header ud_header;
85 u8 header_buf[MLX4_IB_UD_HEADER_SIZE];
86 struct ib_qp *roce_v2_gsi;
87};
88
89enum {
90 MLX4_IB_MIN_SQ_STRIDE = 6,
91 MLX4_IB_CACHE_LINE_SIZE = 64,
92};
93
94enum {
95 MLX4_RAW_QP_MTU = 7,
96 MLX4_RAW_QP_MSGMAX = 31,
97};
98
99#ifndef ETH_ALEN
100#define ETH_ALEN 6
101#endif
102
103static const __be32 mlx4_ib_opcode[] = {
104 [IB_WR_SEND] = cpu_to_be32(MLX4_OPCODE_SEND),
105 [IB_WR_LSO] = cpu_to_be32(MLX4_OPCODE_LSO),
106 [IB_WR_SEND_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_SEND_IMM),
107 [IB_WR_RDMA_WRITE] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
108 [IB_WR_RDMA_WRITE_WITH_IMM] = cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
109 [IB_WR_RDMA_READ] = cpu_to_be32(MLX4_OPCODE_RDMA_READ),
110 [IB_WR_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
111 [IB_WR_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
112 [IB_WR_SEND_WITH_INV] = cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
113 [IB_WR_LOCAL_INV] = cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
114 [IB_WR_REG_MR] = cpu_to_be32(MLX4_OPCODE_FMR),
115 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
116 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
117};
118
119enum mlx4_ib_source_type {
120 MLX4_IB_QP_SRC = 0,
121 MLX4_IB_RWQ_SRC = 1,
122};
123
124static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
125{
126 return container_of(mqp, struct mlx4_ib_sqp, qp);
127}
128
129static int is_tunnel_qp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
130{
131 if (!mlx4_is_master(dev->dev))
132 return 0;
133
134 return qp->mqp.qpn >= dev->dev->phys_caps.base_tunnel_sqpn &&
135 qp->mqp.qpn < dev->dev->phys_caps.base_tunnel_sqpn +
136 8 * MLX4_MFUNC_MAX;
137}
138
139static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
140{
141 int proxy_sqp = 0;
142 int real_sqp = 0;
143 int i;
144
145 real_sqp = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
146 qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
147 qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 3);
148 if (real_sqp)
149 return 1;
150
151 if (mlx4_is_mfunc(dev->dev)) {
152 for (i = 0; i < dev->dev->caps.num_ports; i++) {
153 if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy ||
154 qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp1_proxy) {
155 proxy_sqp = 1;
156 break;
157 }
158 }
159 }
160 if (proxy_sqp)
161 return 1;
162
163 return !!(qp->flags & MLX4_IB_ROCE_V2_GSI_QP);
164}
165
166
167static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
168{
169 int proxy_qp0 = 0;
170 int real_qp0 = 0;
171 int i;
172
173 real_qp0 = ((mlx4_is_master(dev->dev) || !mlx4_is_mfunc(dev->dev)) &&
174 qp->mqp.qpn >= dev->dev->phys_caps.base_sqpn &&
175 qp->mqp.qpn <= dev->dev->phys_caps.base_sqpn + 1);
176 if (real_qp0)
177 return 1;
178
179 if (mlx4_is_mfunc(dev->dev)) {
180 for (i = 0; i < dev->dev->caps.num_ports; i++) {
181 if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy) {
182 proxy_qp0 = 1;
183 break;
184 }
185 }
186 }
187 return proxy_qp0;
188}
189
190static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
191{
192 return mlx4_buf_offset(&qp->buf, offset);
193}
194
195static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
196{
197 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
198}
199
200static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
201{
202 return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
203}
204
205
206
207
208
209
210
211
212
213
214static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
215{
216 __be32 *wqe;
217 int i;
218 int s;
219 int ind;
220 void *buf;
221 __be32 stamp;
222 struct mlx4_wqe_ctrl_seg *ctrl;
223
224 if (qp->sq_max_wqes_per_wr > 1) {
225 s = roundup(size, 1U << qp->sq.wqe_shift);
226 for (i = 0; i < s; i += 64) {
227 ind = (i >> qp->sq.wqe_shift) + n;
228 stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
229 cpu_to_be32(0xffffffff);
230 buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
231 wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
232 *wqe = stamp;
233 }
234 } else {
235 ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
236 s = (ctrl->qpn_vlan.fence_size & 0x3f) << 4;
237 for (i = 64; i < s; i += 64) {
238 wqe = buf + i;
239 *wqe = cpu_to_be32(0xffffffff);
240 }
241 }
242}
243
244static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
245{
246 struct mlx4_wqe_ctrl_seg *ctrl;
247 struct mlx4_wqe_inline_seg *inl;
248 void *wqe;
249 int s;
250
251 ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
252 s = sizeof(struct mlx4_wqe_ctrl_seg);
253
254 if (qp->ibqp.qp_type == IB_QPT_UD) {
255 struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
256 struct mlx4_av *av = (struct mlx4_av *)dgram->av;
257 memset(dgram, 0, sizeof *dgram);
258 av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
259 s += sizeof(struct mlx4_wqe_datagram_seg);
260 }
261
262
263 if (size > s) {
264 inl = wqe + s;
265 inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
266 }
267 ctrl->srcrb_flags = 0;
268 ctrl->qpn_vlan.fence_size = size / 16;
269
270
271
272
273 wmb();
274
275 ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
276 (n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
277
278 stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
279}
280
281
282static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
283{
284 unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
285 if (unlikely(s < qp->sq_max_wqes_per_wr)) {
286 post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
287 ind += s;
288 }
289 return ind;
290}
291
292static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
293{
294 struct ib_event event;
295 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
296
297 if (type == MLX4_EVENT_TYPE_PATH_MIG)
298 to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
299
300 if (ibqp->event_handler) {
301 event.device = ibqp->device;
302 event.element.qp = ibqp;
303 switch (type) {
304 case MLX4_EVENT_TYPE_PATH_MIG:
305 event.event = IB_EVENT_PATH_MIG;
306 break;
307 case MLX4_EVENT_TYPE_COMM_EST:
308 event.event = IB_EVENT_COMM_EST;
309 break;
310 case MLX4_EVENT_TYPE_SQ_DRAINED:
311 event.event = IB_EVENT_SQ_DRAINED;
312 break;
313 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
314 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
315 break;
316 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
317 event.event = IB_EVENT_QP_FATAL;
318 break;
319 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
320 event.event = IB_EVENT_PATH_MIG_ERR;
321 break;
322 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
323 event.event = IB_EVENT_QP_REQ_ERR;
324 break;
325 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
326 event.event = IB_EVENT_QP_ACCESS_ERR;
327 break;
328 default:
329 pr_warn("Unexpected event type %d "
330 "on QP %06x\n", type, qp->qpn);
331 return;
332 }
333
334 ibqp->event_handler(&event, ibqp->qp_context);
335 }
336}
337
338static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
339{
340 pr_warn_ratelimited("Unexpected event type %d on WQ 0x%06x. Events are not supported for WQs\n",
341 type, qp->qpn);
342}
343
344static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
345{
346
347
348
349
350
351
352 switch (type) {
353 case MLX4_IB_QPT_UD:
354 return sizeof (struct mlx4_wqe_ctrl_seg) +
355 sizeof (struct mlx4_wqe_datagram_seg) +
356 ((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
357 case MLX4_IB_QPT_PROXY_SMI_OWNER:
358 case MLX4_IB_QPT_PROXY_SMI:
359 case MLX4_IB_QPT_PROXY_GSI:
360 return sizeof (struct mlx4_wqe_ctrl_seg) +
361 sizeof (struct mlx4_wqe_datagram_seg) + 64;
362 case MLX4_IB_QPT_TUN_SMI_OWNER:
363 case MLX4_IB_QPT_TUN_GSI:
364 return sizeof (struct mlx4_wqe_ctrl_seg) +
365 sizeof (struct mlx4_wqe_datagram_seg);
366
367 case MLX4_IB_QPT_UC:
368 return sizeof (struct mlx4_wqe_ctrl_seg) +
369 sizeof (struct mlx4_wqe_raddr_seg);
370 case MLX4_IB_QPT_RC:
371 return sizeof (struct mlx4_wqe_ctrl_seg) +
372 sizeof (struct mlx4_wqe_masked_atomic_seg) +
373 sizeof (struct mlx4_wqe_raddr_seg);
374 case MLX4_IB_QPT_SMI:
375 case MLX4_IB_QPT_GSI:
376 return sizeof (struct mlx4_wqe_ctrl_seg) +
377 ALIGN(MLX4_IB_UD_HEADER_SIZE +
378 DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
379 MLX4_INLINE_ALIGN) *
380 sizeof (struct mlx4_wqe_inline_seg),
381 sizeof (struct mlx4_wqe_data_seg)) +
382 ALIGN(4 +
383 sizeof (struct mlx4_wqe_inline_seg),
384 sizeof (struct mlx4_wqe_data_seg));
385 default:
386 return sizeof (struct mlx4_wqe_ctrl_seg);
387 }
388}
389
390static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
391 int is_user, int has_rq, struct mlx4_ib_qp *qp,
392 u32 inl_recv_sz)
393{
394
395 if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
396 cap->max_recv_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg))
397 return -EINVAL;
398
399 if (!has_rq) {
400 if (cap->max_recv_wr || inl_recv_sz)
401 return -EINVAL;
402
403 qp->rq.wqe_cnt = qp->rq.max_gs = 0;
404 } else {
405 u32 max_inl_recv_sz = dev->dev->caps.max_rq_sg *
406 sizeof(struct mlx4_wqe_data_seg);
407 u32 wqe_size;
408
409
410 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge ||
411 inl_recv_sz > max_inl_recv_sz))
412 return -EINVAL;
413
414 qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));
415 qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));
416 wqe_size = qp->rq.max_gs * sizeof(struct mlx4_wqe_data_seg);
417 qp->rq.wqe_shift = ilog2(max_t(u32, wqe_size, inl_recv_sz));
418 }
419
420
421 if (is_user) {
422 cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
423 cap->max_recv_sge = qp->rq.max_gs;
424 } else {
425 cap->max_recv_wr = qp->rq.max_post =
426 min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt);
427 cap->max_recv_sge = min(qp->rq.max_gs,
428 min(dev->dev->caps.max_sq_sg,
429 dev->dev->caps.max_rq_sg));
430 }
431
432 return 0;
433}
434
435static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
436 enum mlx4_ib_qp_type type, struct mlx4_ib_qp *qp,
437 bool shrink_wqe)
438{
439 int s;
440
441
442 if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) ||
443 cap->max_send_sge > min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) ||
444 cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
445 sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
446 return -EINVAL;
447
448
449
450
451
452 if ((type == MLX4_IB_QPT_SMI || type == MLX4_IB_QPT_GSI ||
453 type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) &&
454 cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
455 return -EINVAL;
456
457 s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
458 cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
459 send_wqe_overhead(type, qp->flags);
460
461 if (s > dev->dev->caps.max_sq_desc_sz)
462 return -EINVAL;
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495 if (shrink_wqe && dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
496 qp->sq_signal_bits && BITS_PER_LONG == 64 &&
497 type != MLX4_IB_QPT_SMI && type != MLX4_IB_QPT_GSI &&
498 !(type & (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_PROXY_SMI |
499 MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER)))
500 qp->sq.wqe_shift = ilog2(64);
501 else
502 qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
503
504 for (;;) {
505 qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
506
507
508
509
510
511 qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
512 qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
513 qp->sq_max_wqes_per_wr +
514 qp->sq_spare_wqes);
515
516 if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
517 break;
518
519 if (qp->sq_max_wqes_per_wr <= 1)
520 return -EINVAL;
521
522 ++qp->sq.wqe_shift;
523 }
524
525 qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
526 (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
527 send_wqe_overhead(type, qp->flags)) /
528 sizeof (struct mlx4_wqe_data_seg);
529
530 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
531 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
532 if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
533 qp->rq.offset = 0;
534 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
535 } else {
536 qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
537 qp->sq.offset = 0;
538 }
539
540 cap->max_send_wr = qp->sq.max_post =
541 (qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
542 cap->max_send_sge = min(qp->sq.max_gs,
543 min(dev->dev->caps.max_sq_sg,
544 dev->dev->caps.max_rq_sg));
545
546 cap->max_inline_data = 0;
547
548 return 0;
549}
550
551static int set_user_sq_size(struct mlx4_ib_dev *dev,
552 struct mlx4_ib_qp *qp,
553 struct mlx4_ib_create_qp *ucmd)
554{
555
556 if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes ||
557 ucmd->log_sq_stride >
558 ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
559 ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
560 return -EINVAL;
561
562 qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
563 qp->sq.wqe_shift = ucmd->log_sq_stride;
564
565 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
566 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
567
568 return 0;
569}
570
571static int alloc_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
572{
573 int i;
574
575 qp->sqp_proxy_rcv =
576 kmalloc(sizeof (struct mlx4_ib_buf) * qp->rq.wqe_cnt,
577 GFP_KERNEL);
578 if (!qp->sqp_proxy_rcv)
579 return -ENOMEM;
580 for (i = 0; i < qp->rq.wqe_cnt; i++) {
581 qp->sqp_proxy_rcv[i].addr =
582 kmalloc(sizeof (struct mlx4_ib_proxy_sqp_hdr),
583 GFP_KERNEL);
584 if (!qp->sqp_proxy_rcv[i].addr)
585 goto err;
586 qp->sqp_proxy_rcv[i].map =
587 ib_dma_map_single(dev, qp->sqp_proxy_rcv[i].addr,
588 sizeof (struct mlx4_ib_proxy_sqp_hdr),
589 DMA_FROM_DEVICE);
590 if (ib_dma_mapping_error(dev, qp->sqp_proxy_rcv[i].map)) {
591 kfree(qp->sqp_proxy_rcv[i].addr);
592 goto err;
593 }
594 }
595 return 0;
596
597err:
598 while (i > 0) {
599 --i;
600 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
601 sizeof (struct mlx4_ib_proxy_sqp_hdr),
602 DMA_FROM_DEVICE);
603 kfree(qp->sqp_proxy_rcv[i].addr);
604 }
605 kfree(qp->sqp_proxy_rcv);
606 qp->sqp_proxy_rcv = NULL;
607 return -ENOMEM;
608}
609
610static void free_proxy_bufs(struct ib_device *dev, struct mlx4_ib_qp *qp)
611{
612 int i;
613
614 for (i = 0; i < qp->rq.wqe_cnt; i++) {
615 ib_dma_unmap_single(dev, qp->sqp_proxy_rcv[i].map,
616 sizeof (struct mlx4_ib_proxy_sqp_hdr),
617 DMA_FROM_DEVICE);
618 kfree(qp->sqp_proxy_rcv[i].addr);
619 }
620 kfree(qp->sqp_proxy_rcv);
621}
622
623static int qp_has_rq(struct ib_qp_init_attr *attr)
624{
625 if (attr->qp_type == IB_QPT_XRC_INI || attr->qp_type == IB_QPT_XRC_TGT)
626 return 0;
627
628 return !attr->srq;
629}
630
631static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
632{
633 int i;
634 for (i = 0; i < dev->caps.num_ports; i++) {
635 if (qpn == dev->caps.spec_qps[i].qp0_proxy)
636 return !!dev->caps.spec_qps[i].qp0_qkey;
637 }
638 return 0;
639}
640
641static void mlx4_ib_free_qp_counter(struct mlx4_ib_dev *dev,
642 struct mlx4_ib_qp *qp)
643{
644 mutex_lock(&dev->counters_table[qp->port - 1].mutex);
645 mlx4_counter_free(dev->dev, qp->counter_index->index);
646 list_del(&qp->counter_index->list);
647 mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
648
649 kfree(qp->counter_index);
650 qp->counter_index = NULL;
651}
652
653static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,
654 struct ib_qp_init_attr *init_attr,
655 struct mlx4_ib_create_qp_rss *ucmd)
656{
657 rss_ctx->base_qpn_tbl_sz = init_attr->rwq_ind_tbl->ind_tbl[0]->wq_num |
658 (init_attr->rwq_ind_tbl->log_ind_tbl_size << 24);
659
660 if ((ucmd->rx_hash_function == MLX4_IB_RX_HASH_FUNC_TOEPLITZ) &&
661 (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP)) {
662 memcpy(rss_ctx->rss_key, ucmd->rx_hash_key,
663 MLX4_EN_RSS_KEY_SIZE);
664 } else {
665 pr_debug("RX Hash function is not supported\n");
666 return (-EOPNOTSUPP);
667 }
668
669 if (ucmd->rx_hash_fields_mask & ~(MLX4_IB_RX_HASH_SRC_IPV4 |
670 MLX4_IB_RX_HASH_DST_IPV4 |
671 MLX4_IB_RX_HASH_SRC_IPV6 |
672 MLX4_IB_RX_HASH_DST_IPV6 |
673 MLX4_IB_RX_HASH_SRC_PORT_TCP |
674 MLX4_IB_RX_HASH_DST_PORT_TCP |
675 MLX4_IB_RX_HASH_SRC_PORT_UDP |
676 MLX4_IB_RX_HASH_DST_PORT_UDP)) {
677 pr_debug("RX Hash fields_mask has unsupported mask (0x%llx)\n",
678 ucmd->rx_hash_fields_mask);
679 return (-EOPNOTSUPP);
680 }
681
682 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) &&
683 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
684 rss_ctx->flags = MLX4_RSS_IPV4;
685 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) ||
686 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
687 pr_debug("RX Hash fields_mask is not supported - both IPv4 SRC and DST must be set\n");
688 return (-EOPNOTSUPP);
689 }
690
691 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) &&
692 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
693 rss_ctx->flags |= MLX4_RSS_IPV6;
694 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) ||
695 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
696 pr_debug("RX Hash fields_mask is not supported - both IPv6 SRC and DST must be set\n");
697 return (-EOPNOTSUPP);
698 }
699
700 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) &&
701 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
702 if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS)) {
703 pr_debug("RX Hash fields_mask for UDP is not supported\n");
704 return (-EOPNOTSUPP);
705 }
706
707 if (rss_ctx->flags & MLX4_RSS_IPV4)
708 rss_ctx->flags |= MLX4_RSS_UDP_IPV4;
709 if (rss_ctx->flags & MLX4_RSS_IPV6)
710 rss_ctx->flags |= MLX4_RSS_UDP_IPV6;
711 if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
712 pr_debug("RX Hash fields_mask is not supported - UDP must be set with IPv4 or IPv6\n");
713 return (-EOPNOTSUPP);
714 }
715 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) ||
716 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
717 pr_debug("RX Hash fields_mask is not supported - both UDP SRC and DST must be set\n");
718 return (-EOPNOTSUPP);
719 }
720
721 if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) &&
722 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
723 if (rss_ctx->flags & MLX4_RSS_IPV4)
724 rss_ctx->flags |= MLX4_RSS_TCP_IPV4;
725 if (rss_ctx->flags & MLX4_RSS_IPV6)
726 rss_ctx->flags |= MLX4_RSS_TCP_IPV6;
727 if (!(rss_ctx->flags & (MLX4_RSS_IPV6 | MLX4_RSS_IPV4))) {
728 pr_debug("RX Hash fields_mask is not supported - TCP must be set with IPv4 or IPv6\n");
729 return (-EOPNOTSUPP);
730 }
731 } else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) ||
732 (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
733 pr_debug("RX Hash fields_mask is not supported - both TCP SRC and DST must be set\n");
734 return (-EOPNOTSUPP);
735 }
736
737 return 0;
738}
739
740static int create_qp_rss(struct mlx4_ib_dev *dev, struct ib_pd *ibpd,
741 struct ib_qp_init_attr *init_attr,
742 struct mlx4_ib_create_qp_rss *ucmd,
743 struct mlx4_ib_qp *qp)
744{
745 int qpn;
746 int err;
747
748 qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
749
750 err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn, 0, qp->mqp.usage);
751 if (err)
752 return err;
753
754 err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
755 if (err)
756 goto err_qpn;
757
758 mutex_init(&qp->mutex);
759
760 INIT_LIST_HEAD(&qp->gid_list);
761 INIT_LIST_HEAD(&qp->steering_rules);
762
763 qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
764 qp->state = IB_QPS_RESET;
765
766
767 qp->sq_no_prefetch = 1;
768 qp->sq.wqe_cnt = 1;
769 qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
770 qp->buf_size = qp->sq.wqe_cnt << MLX4_IB_MIN_SQ_STRIDE;
771 qp->mtt = (to_mqp(
772 (struct ib_qp *)init_attr->rwq_ind_tbl->ind_tbl[0]))->mtt;
773
774 qp->rss_ctx = kzalloc(sizeof(*qp->rss_ctx), GFP_KERNEL);
775 if (!qp->rss_ctx) {
776 err = -ENOMEM;
777 goto err_qp_alloc;
778 }
779
780 err = set_qp_rss(dev, qp->rss_ctx, init_attr, ucmd);
781 if (err)
782 goto err;
783
784 return 0;
785
786err:
787 kfree(qp->rss_ctx);
788
789err_qp_alloc:
790 mlx4_qp_remove(dev->dev, &qp->mqp);
791 mlx4_qp_free(dev->dev, &qp->mqp);
792
793err_qpn:
794 mlx4_qp_release_range(dev->dev, qpn, 1);
795 return err;
796}
797
798static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd,
799 struct ib_qp_init_attr *init_attr,
800 struct ib_udata *udata)
801{
802 struct mlx4_ib_qp *qp;
803 struct mlx4_ib_create_qp_rss ucmd = {};
804 size_t required_cmd_sz;
805 int err;
806
807 if (!udata) {
808 pr_debug("RSS QP with NULL udata\n");
809 return ERR_PTR(-EINVAL);
810 }
811
812 if (udata->outlen)
813 return ERR_PTR(-EOPNOTSUPP);
814
815 required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
816 sizeof(ucmd.reserved1);
817 if (udata->inlen < required_cmd_sz) {
818 pr_debug("invalid inlen\n");
819 return ERR_PTR(-EINVAL);
820 }
821
822 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
823 pr_debug("copy failed\n");
824 return ERR_PTR(-EFAULT);
825 }
826
827 if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
828 return ERR_PTR(-EOPNOTSUPP);
829
830 if (ucmd.comp_mask || ucmd.reserved1)
831 return ERR_PTR(-EOPNOTSUPP);
832
833 if (udata->inlen > sizeof(ucmd) &&
834 !ib_is_udata_cleared(udata, sizeof(ucmd),
835 udata->inlen - sizeof(ucmd))) {
836 pr_debug("inlen is not supported\n");
837 return ERR_PTR(-EOPNOTSUPP);
838 }
839
840 if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
841 pr_debug("RSS QP with unsupported QP type %d\n",
842 init_attr->qp_type);
843 return ERR_PTR(-EOPNOTSUPP);
844 }
845
846 if (init_attr->create_flags) {
847 pr_debug("RSS QP doesn't support create flags\n");
848 return ERR_PTR(-EOPNOTSUPP);
849 }
850
851 if (init_attr->send_cq || init_attr->cap.max_send_wr) {
852 pr_debug("RSS QP with unsupported send attributes\n");
853 return ERR_PTR(-EOPNOTSUPP);
854 }
855
856 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
857 if (!qp)
858 return ERR_PTR(-ENOMEM);
859
860 qp->pri.vid = 0xFFFF;
861 qp->alt.vid = 0xFFFF;
862
863 err = create_qp_rss(to_mdev(pd->device), pd, init_attr, &ucmd, qp);
864 if (err) {
865 kfree(qp);
866 return ERR_PTR(err);
867 }
868
869 qp->ibqp.qp_num = qp->mqp.qpn;
870
871 return &qp->ibqp;
872}
873
874
875
876
877
878
879static int mlx4_ib_alloc_wqn(struct mlx4_ib_ucontext *context,
880 struct mlx4_ib_qp *qp, int range_size, int *wqn)
881{
882 struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
883 struct mlx4_wqn_range *range;
884 int err = 0;
885
886 mutex_lock(&context->wqn_ranges_mutex);
887
888 range = list_first_entry_or_null(&context->wqn_ranges_list,
889 struct mlx4_wqn_range, list);
890
891 if (!range || (range->refcount == range->size) || range->dirty) {
892 range = kzalloc(sizeof(*range), GFP_KERNEL);
893 if (!range) {
894 err = -ENOMEM;
895 goto out;
896 }
897
898 err = mlx4_qp_reserve_range(dev->dev, range_size,
899 range_size, &range->base_wqn, 0,
900 qp->mqp.usage);
901 if (err) {
902 kfree(range);
903 goto out;
904 }
905
906 range->size = range_size;
907 list_add(&range->list, &context->wqn_ranges_list);
908 } else if (range_size != 1) {
909
910
911
912
913 err = -EINVAL;
914 goto out;
915 }
916
917 qp->wqn_range = range;
918
919 *wqn = range->base_wqn + range->refcount;
920
921 range->refcount++;
922
923out:
924 mutex_unlock(&context->wqn_ranges_mutex);
925
926 return err;
927}
928
929static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context,
930 struct mlx4_ib_qp *qp, bool dirty_release)
931{
932 struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
933 struct mlx4_wqn_range *range;
934
935 mutex_lock(&context->wqn_ranges_mutex);
936
937 range = qp->wqn_range;
938
939 range->refcount--;
940 if (!range->refcount) {
941 mlx4_qp_release_range(dev->dev, range->base_wqn,
942 range->size);
943 list_del(&range->list);
944 kfree(range);
945 } else if (dirty_release) {
946
947
948
949
950
951 range->dirty = 1;
952 }
953
954 mutex_unlock(&context->wqn_ranges_mutex);
955}
956
957static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
958 enum mlx4_ib_source_type src,
959 struct ib_qp_init_attr *init_attr,
960 struct ib_udata *udata, int sqpn,
961 struct mlx4_ib_qp **caller_qp)
962{
963 int qpn;
964 int err;
965 struct ib_qp_cap backup_cap;
966 struct mlx4_ib_sqp *sqp = NULL;
967 struct mlx4_ib_qp *qp;
968 enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
969 struct mlx4_ib_cq *mcq;
970 unsigned long flags;
971 int range_size = 0;
972
973
974 if (sqpn) {
975 if (mlx4_is_mfunc(dev->dev) &&
976 (!mlx4_is_master(dev->dev) ||
977 !(init_attr->create_flags & MLX4_IB_SRIOV_SQP))) {
978 if (init_attr->qp_type == IB_QPT_GSI)
979 qp_type = MLX4_IB_QPT_PROXY_GSI;
980 else {
981 if (mlx4_is_master(dev->dev) ||
982 qp0_enabled_vf(dev->dev, sqpn))
983 qp_type = MLX4_IB_QPT_PROXY_SMI_OWNER;
984 else
985 qp_type = MLX4_IB_QPT_PROXY_SMI;
986 }
987 }
988 qpn = sqpn;
989
990 init_attr->cap.max_recv_sge++;
991 } else if (init_attr->create_flags & MLX4_IB_SRIOV_TUNNEL_QP) {
992 struct mlx4_ib_qp_tunnel_init_attr *tnl_init =
993 container_of(init_attr,
994 struct mlx4_ib_qp_tunnel_init_attr, init_attr);
995 if ((tnl_init->proxy_qp_type != IB_QPT_SMI &&
996 tnl_init->proxy_qp_type != IB_QPT_GSI) ||
997 !mlx4_is_master(dev->dev))
998 return -EINVAL;
999 if (tnl_init->proxy_qp_type == IB_QPT_GSI)
1000 qp_type = MLX4_IB_QPT_TUN_GSI;
1001 else if (tnl_init->slave == mlx4_master_func_num(dev->dev) ||
1002 mlx4_vf_smi_enabled(dev->dev, tnl_init->slave,
1003 tnl_init->port))
1004 qp_type = MLX4_IB_QPT_TUN_SMI_OWNER;
1005 else
1006 qp_type = MLX4_IB_QPT_TUN_SMI;
1007
1008
1009 qpn = dev->dev->phys_caps.base_tunnel_sqpn + 8 * tnl_init->slave
1010 + tnl_init->proxy_qp_type * 2 + tnl_init->port - 1;
1011 sqpn = qpn;
1012 }
1013
1014 if (!*caller_qp) {
1015 if (qp_type == MLX4_IB_QPT_SMI || qp_type == MLX4_IB_QPT_GSI ||
1016 (qp_type & (MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_SMI_OWNER |
1017 MLX4_IB_QPT_PROXY_GSI | MLX4_IB_QPT_TUN_SMI_OWNER))) {
1018 sqp = kzalloc(sizeof(struct mlx4_ib_sqp), GFP_KERNEL);
1019 if (!sqp)
1020 return -ENOMEM;
1021 qp = &sqp->qp;
1022 qp->pri.vid = 0xFFFF;
1023 qp->alt.vid = 0xFFFF;
1024 } else {
1025 qp = kzalloc(sizeof(struct mlx4_ib_qp), GFP_KERNEL);
1026 if (!qp)
1027 return -ENOMEM;
1028 qp->pri.vid = 0xFFFF;
1029 qp->alt.vid = 0xFFFF;
1030 }
1031 } else
1032 qp = *caller_qp;
1033
1034 qp->mlx4_ib_qp_type = qp_type;
1035
1036 mutex_init(&qp->mutex);
1037 spin_lock_init(&qp->sq.lock);
1038 spin_lock_init(&qp->rq.lock);
1039 INIT_LIST_HEAD(&qp->gid_list);
1040 INIT_LIST_HEAD(&qp->steering_rules);
1041
1042 qp->state = IB_QPS_RESET;
1043 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1044 qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
1045
1046
1047 if (pd->uobject) {
1048 union {
1049 struct mlx4_ib_create_qp qp;
1050 struct mlx4_ib_create_wq wq;
1051 } ucmd;
1052 size_t copy_len;
1053 int shift;
1054 int n;
1055
1056 copy_len = (src == MLX4_IB_QP_SRC) ?
1057 sizeof(struct mlx4_ib_create_qp) :
1058 min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
1059
1060 if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
1061 err = -EFAULT;
1062 goto err;
1063 }
1064
1065 if (src == MLX4_IB_RWQ_SRC) {
1066 if (ucmd.wq.comp_mask || ucmd.wq.reserved[0] ||
1067 ucmd.wq.reserved[1] || ucmd.wq.reserved[2]) {
1068 pr_debug("user command isn't supported\n");
1069 err = -EOPNOTSUPP;
1070 goto err;
1071 }
1072
1073 if (ucmd.wq.log_range_size >
1074 ilog2(dev->dev->caps.max_rss_tbl_sz)) {
1075 pr_debug("WQN range size must be equal or smaller than %d\n",
1076 dev->dev->caps.max_rss_tbl_sz);
1077 err = -EOPNOTSUPP;
1078 goto err;
1079 }
1080 range_size = 1 << ucmd.wq.log_range_size;
1081 } else {
1082 qp->inl_recv_sz = ucmd.qp.inl_recv_sz;
1083 }
1084
1085 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
1086 qp_has_rq(init_attr), qp, qp->inl_recv_sz);
1087 if (err)
1088 goto err;
1089
1090 if (src == MLX4_IB_QP_SRC) {
1091 qp->sq_no_prefetch = ucmd.qp.sq_no_prefetch;
1092
1093 err = set_user_sq_size(dev, qp,
1094 (struct mlx4_ib_create_qp *)
1095 &ucmd);
1096 if (err)
1097 goto err;
1098 } else {
1099 qp->sq_no_prefetch = 1;
1100 qp->sq.wqe_cnt = 1;
1101 qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
1102
1103
1104
1105 qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
1106 (qp->sq.wqe_cnt << qp->sq.wqe_shift);
1107 }
1108
1109 qp->umem = ib_umem_get(pd->uobject->context,
1110 (src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :
1111 ucmd.wq.buf_addr, qp->buf_size, 0, 0);
1112 if (IS_ERR(qp->umem)) {
1113 err = PTR_ERR(qp->umem);
1114 goto err;
1115 }
1116
1117 n = ib_umem_page_count(qp->umem);
1118 shift = mlx4_ib_umem_calc_optimal_mtt_size(qp->umem, 0, &n);
1119 err = mlx4_mtt_init(dev->dev, n, shift, &qp->mtt);
1120
1121 if (err)
1122 goto err_buf;
1123
1124 err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
1125 if (err)
1126 goto err_mtt;
1127
1128 if (qp_has_rq(init_attr)) {
1129 err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
1130 (src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
1131 ucmd.wq.db_addr, &qp->db);
1132 if (err)
1133 goto err_mtt;
1134 }
1135 qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
1136 } else {
1137 err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
1138 qp_has_rq(init_attr), qp, 0);
1139 if (err)
1140 goto err;
1141
1142 qp->sq_no_prefetch = 0;
1143
1144 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
1145 qp->flags |= MLX4_IB_QP_LSO;
1146
1147 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1148 if (dev->steering_support ==
1149 MLX4_STEERING_MODE_DEVICE_MANAGED)
1150 qp->flags |= MLX4_IB_QP_NETIF;
1151 else
1152 goto err;
1153 }
1154
1155 memcpy(&backup_cap, &init_attr->cap, sizeof(backup_cap));
1156 err = set_kernel_sq_size(dev, &init_attr->cap,
1157 qp_type, qp, true);
1158 if (err)
1159 goto err;
1160
1161 if (qp_has_rq(init_attr)) {
1162 err = mlx4_db_alloc(dev->dev, &qp->db, 0);
1163 if (err)
1164 goto err;
1165
1166 *qp->db.db = 0;
1167 }
1168
1169 if (mlx4_buf_alloc(dev->dev, qp->buf_size, qp->buf_size,
1170 &qp->buf)) {
1171 memcpy(&init_attr->cap, &backup_cap,
1172 sizeof(backup_cap));
1173 err = set_kernel_sq_size(dev, &init_attr->cap, qp_type,
1174 qp, false);
1175 if (err)
1176 goto err_db;
1177
1178 if (mlx4_buf_alloc(dev->dev, qp->buf_size,
1179 PAGE_SIZE * 2, &qp->buf)) {
1180 err = -ENOMEM;
1181 goto err_db;
1182 }
1183 }
1184
1185 err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
1186 &qp->mtt);
1187 if (err)
1188 goto err_buf;
1189
1190 err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
1191 if (err)
1192 goto err_mtt;
1193
1194 qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1195 sizeof(u64), GFP_KERNEL);
1196 qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1197 sizeof(u64), GFP_KERNEL);
1198 if (!qp->sq.wrid || !qp->rq.wrid) {
1199 err = -ENOMEM;
1200 goto err_wrid;
1201 }
1202 qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
1203 }
1204
1205 if (sqpn) {
1206 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1207 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
1208 if (alloc_proxy_bufs(pd->device, qp)) {
1209 err = -ENOMEM;
1210 goto err_wrid;
1211 }
1212 }
1213 } else if (src == MLX4_IB_RWQ_SRC) {
1214 err = mlx4_ib_alloc_wqn(to_mucontext(pd->uobject->context), qp,
1215 range_size, &qpn);
1216 if (err)
1217 goto err_wrid;
1218 } else {
1219
1220
1221
1222 if (init_attr->qp_type == IB_QPT_RAW_PACKET)
1223 err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
1224 (init_attr->cap.max_send_wr ?
1225 MLX4_RESERVE_ETH_BF_QP : 0) |
1226 (init_attr->cap.max_recv_wr ?
1227 MLX4_RESERVE_A0_QP : 0),
1228 qp->mqp.usage);
1229 else
1230 if (qp->flags & MLX4_IB_QP_NETIF)
1231 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
1232 else
1233 err = mlx4_qp_reserve_range(dev->dev, 1, 1,
1234 &qpn, 0, qp->mqp.usage);
1235 if (err)
1236 goto err_proxy;
1237 }
1238
1239 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
1240 qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1241
1242 err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
1243 if (err)
1244 goto err_qpn;
1245
1246 if (init_attr->qp_type == IB_QPT_XRC_TGT)
1247 qp->mqp.qpn |= (1 << 23);
1248
1249
1250
1251
1252
1253
1254 qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
1255
1256 qp->mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :
1257 mlx4_ib_wq_event;
1258
1259 if (!*caller_qp)
1260 *caller_qp = qp;
1261
1262 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1263 mlx4_ib_lock_cqs(to_mcq(init_attr->send_cq),
1264 to_mcq(init_attr->recv_cq));
1265
1266
1267
1268 list_add_tail(&qp->qps_list, &dev->qp_list);
1269
1270
1271
1272 mcq = to_mcq(init_attr->send_cq);
1273 list_add_tail(&qp->cq_send_list, &mcq->send_qp_list);
1274 mcq = to_mcq(init_attr->recv_cq);
1275 list_add_tail(&qp->cq_recv_list, &mcq->recv_qp_list);
1276 mlx4_ib_unlock_cqs(to_mcq(init_attr->send_cq),
1277 to_mcq(init_attr->recv_cq));
1278 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1279 return 0;
1280
1281err_qpn:
1282 if (!sqpn) {
1283 if (qp->flags & MLX4_IB_QP_NETIF)
1284 mlx4_ib_steer_qp_free(dev, qpn, 1);
1285 else if (src == MLX4_IB_RWQ_SRC)
1286 mlx4_ib_release_wqn(to_mucontext(pd->uobject->context),
1287 qp, 0);
1288 else
1289 mlx4_qp_release_range(dev->dev, qpn, 1);
1290 }
1291err_proxy:
1292 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
1293 free_proxy_bufs(pd->device, qp);
1294err_wrid:
1295 if (pd->uobject) {
1296 if (qp_has_rq(init_attr))
1297 mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
1298 } else {
1299 kvfree(qp->sq.wrid);
1300 kvfree(qp->rq.wrid);
1301 }
1302
1303err_mtt:
1304 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1305
1306err_buf:
1307 if (pd->uobject)
1308 ib_umem_release(qp->umem);
1309 else
1310 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1311
1312err_db:
1313 if (!pd->uobject && qp_has_rq(init_attr))
1314 mlx4_db_free(dev->dev, &qp->db);
1315
1316err:
1317 if (sqp)
1318 kfree(sqp);
1319 else if (!*caller_qp)
1320 kfree(qp);
1321 return err;
1322}
1323
1324static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
1325{
1326 switch (state) {
1327 case IB_QPS_RESET: return MLX4_QP_STATE_RST;
1328 case IB_QPS_INIT: return MLX4_QP_STATE_INIT;
1329 case IB_QPS_RTR: return MLX4_QP_STATE_RTR;
1330 case IB_QPS_RTS: return MLX4_QP_STATE_RTS;
1331 case IB_QPS_SQD: return MLX4_QP_STATE_SQD;
1332 case IB_QPS_SQE: return MLX4_QP_STATE_SQER;
1333 case IB_QPS_ERR: return MLX4_QP_STATE_ERR;
1334 default: return -1;
1335 }
1336}
1337
1338static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1339 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1340{
1341 if (send_cq == recv_cq) {
1342 spin_lock(&send_cq->lock);
1343 __acquire(&recv_cq->lock);
1344 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1345 spin_lock(&send_cq->lock);
1346 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1347 } else {
1348 spin_lock(&recv_cq->lock);
1349 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1350 }
1351}
1352
1353static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
1354 __releases(&send_cq->lock) __releases(&recv_cq->lock)
1355{
1356 if (send_cq == recv_cq) {
1357 __release(&recv_cq->lock);
1358 spin_unlock(&send_cq->lock);
1359 } else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1360 spin_unlock(&recv_cq->lock);
1361 spin_unlock(&send_cq->lock);
1362 } else {
1363 spin_unlock(&send_cq->lock);
1364 spin_unlock(&recv_cq->lock);
1365 }
1366}
1367
1368static void del_gid_entries(struct mlx4_ib_qp *qp)
1369{
1370 struct mlx4_ib_gid_entry *ge, *tmp;
1371
1372 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1373 list_del(&ge->list);
1374 kfree(ge);
1375 }
1376}
1377
1378static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
1379{
1380 if (qp->ibqp.qp_type == IB_QPT_XRC_TGT)
1381 return to_mpd(to_mxrcd(qp->ibqp.xrcd)->pd);
1382 else
1383 return to_mpd(qp->ibqp.pd);
1384}
1385
1386static void get_cqs(struct mlx4_ib_qp *qp, enum mlx4_ib_source_type src,
1387 struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
1388{
1389 switch (qp->ibqp.qp_type) {
1390 case IB_QPT_XRC_TGT:
1391 *send_cq = to_mcq(to_mxrcd(qp->ibqp.xrcd)->cq);
1392 *recv_cq = *send_cq;
1393 break;
1394 case IB_QPT_XRC_INI:
1395 *send_cq = to_mcq(qp->ibqp.send_cq);
1396 *recv_cq = *send_cq;
1397 break;
1398 default:
1399 *recv_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.recv_cq) :
1400 to_mcq(qp->ibwq.cq);
1401 *send_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.send_cq) :
1402 *recv_cq;
1403 break;
1404 }
1405}
1406
1407static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1408{
1409 if (qp->state != IB_QPS_RESET) {
1410 int i;
1411
1412 for (i = 0; i < (1 << qp->ibqp.rwq_ind_tbl->log_ind_tbl_size);
1413 i++) {
1414 struct ib_wq *ibwq = qp->ibqp.rwq_ind_tbl->ind_tbl[i];
1415 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
1416
1417 mutex_lock(&wq->mutex);
1418
1419 wq->rss_usecnt--;
1420
1421 mutex_unlock(&wq->mutex);
1422 }
1423
1424 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1425 MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1426 pr_warn("modify QP %06x to RESET failed.\n",
1427 qp->mqp.qpn);
1428 }
1429
1430 mlx4_qp_remove(dev->dev, &qp->mqp);
1431 mlx4_qp_free(dev->dev, &qp->mqp);
1432 mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1433 del_gid_entries(qp);
1434 kfree(qp->rss_ctx);
1435}
1436
1437static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
1438 enum mlx4_ib_source_type src, int is_user)
1439{
1440 struct mlx4_ib_cq *send_cq, *recv_cq;
1441 unsigned long flags;
1442
1443 if (qp->state != IB_QPS_RESET) {
1444 if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
1445 MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
1446 pr_warn("modify QP %06x to RESET failed.\n",
1447 qp->mqp.qpn);
1448 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
1449 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
1450 qp->pri.smac = 0;
1451 qp->pri.smac_port = 0;
1452 }
1453 if (qp->alt.smac) {
1454 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
1455 qp->alt.smac = 0;
1456 }
1457 if (qp->pri.vid < 0x1000) {
1458 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
1459 qp->pri.vid = 0xFFFF;
1460 qp->pri.candidate_vid = 0xFFFF;
1461 qp->pri.update_vid = 0;
1462 }
1463 if (qp->alt.vid < 0x1000) {
1464 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
1465 qp->alt.vid = 0xFFFF;
1466 qp->alt.candidate_vid = 0xFFFF;
1467 qp->alt.update_vid = 0;
1468 }
1469 }
1470
1471 get_cqs(qp, src, &send_cq, &recv_cq);
1472
1473 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1474 mlx4_ib_lock_cqs(send_cq, recv_cq);
1475
1476
1477 list_del(&qp->qps_list);
1478 list_del(&qp->cq_send_list);
1479 list_del(&qp->cq_recv_list);
1480 if (!is_user) {
1481 __mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
1482 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
1483 if (send_cq != recv_cq)
1484 __mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
1485 }
1486
1487 mlx4_qp_remove(dev->dev, &qp->mqp);
1488
1489 mlx4_ib_unlock_cqs(send_cq, recv_cq);
1490 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1491
1492 mlx4_qp_free(dev->dev, &qp->mqp);
1493
1494 if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
1495 if (qp->flags & MLX4_IB_QP_NETIF)
1496 mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
1497 else if (src == MLX4_IB_RWQ_SRC)
1498 mlx4_ib_release_wqn(to_mucontext(
1499 qp->ibwq.uobject->context), qp, 1);
1500 else
1501 mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
1502 }
1503
1504 mlx4_mtt_cleanup(dev->dev, &qp->mtt);
1505
1506 if (is_user) {
1507 if (qp->rq.wqe_cnt) {
1508 struct mlx4_ib_ucontext *mcontext = !src ?
1509 to_mucontext(qp->ibqp.uobject->context) :
1510 to_mucontext(qp->ibwq.uobject->context);
1511 mlx4_ib_db_unmap_user(mcontext, &qp->db);
1512 }
1513 ib_umem_release(qp->umem);
1514 } else {
1515 kvfree(qp->sq.wrid);
1516 kvfree(qp->rq.wrid);
1517 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
1518 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI))
1519 free_proxy_bufs(&dev->ib_dev, qp);
1520 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
1521 if (qp->rq.wqe_cnt)
1522 mlx4_db_free(dev->dev, &qp->db);
1523 }
1524
1525 del_gid_entries(qp);
1526}
1527
1528static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
1529{
1530
1531 if (!mlx4_is_mfunc(dev->dev) ||
1532 (mlx4_is_master(dev->dev) &&
1533 attr->create_flags & MLX4_IB_SRIOV_SQP)) {
1534 return dev->dev->phys_caps.base_sqpn +
1535 (attr->qp_type == IB_QPT_SMI ? 0 : 2) +
1536 attr->port_num - 1;
1537 }
1538
1539 if (attr->qp_type == IB_QPT_SMI)
1540 return dev->dev->caps.spec_qps[attr->port_num - 1].qp0_proxy;
1541 else
1542 return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
1543}
1544
1545static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
1546 struct ib_qp_init_attr *init_attr,
1547 struct ib_udata *udata)
1548{
1549 struct mlx4_ib_qp *qp = NULL;
1550 int err;
1551 int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1552 u16 xrcdn = 0;
1553
1554 if (init_attr->rwq_ind_tbl)
1555 return _mlx4_ib_create_qp_rss(pd, init_attr, udata);
1556
1557
1558
1559
1560
1561 if (init_attr->create_flags & ~(MLX4_IB_QP_LSO |
1562 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK |
1563 MLX4_IB_SRIOV_TUNNEL_QP |
1564 MLX4_IB_SRIOV_SQP |
1565 MLX4_IB_QP_NETIF |
1566 MLX4_IB_QP_CREATE_ROCE_V2_GSI))
1567 return ERR_PTR(-EINVAL);
1568
1569 if (init_attr->create_flags & IB_QP_CREATE_NETIF_QP) {
1570 if (init_attr->qp_type != IB_QPT_UD)
1571 return ERR_PTR(-EINVAL);
1572 }
1573
1574 if (init_attr->create_flags) {
1575 if (udata && init_attr->create_flags & ~(sup_u_create_flags))
1576 return ERR_PTR(-EINVAL);
1577
1578 if ((init_attr->create_flags & ~(MLX4_IB_SRIOV_SQP |
1579 MLX4_IB_QP_CREATE_ROCE_V2_GSI |
1580 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) &&
1581 init_attr->qp_type != IB_QPT_UD) ||
1582 (init_attr->create_flags & MLX4_IB_SRIOV_SQP &&
1583 init_attr->qp_type > IB_QPT_GSI) ||
1584 (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI &&
1585 init_attr->qp_type != IB_QPT_GSI))
1586 return ERR_PTR(-EINVAL);
1587 }
1588
1589 switch (init_attr->qp_type) {
1590 case IB_QPT_XRC_TGT:
1591 pd = to_mxrcd(init_attr->xrcd)->pd;
1592 xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
1593 init_attr->send_cq = to_mxrcd(init_attr->xrcd)->cq;
1594
1595 case IB_QPT_XRC_INI:
1596 if (!(to_mdev(pd->device)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
1597 return ERR_PTR(-ENOSYS);
1598 init_attr->recv_cq = init_attr->send_cq;
1599
1600 case IB_QPT_RC:
1601 case IB_QPT_UC:
1602 case IB_QPT_RAW_PACKET:
1603 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1604 if (!qp)
1605 return ERR_PTR(-ENOMEM);
1606 qp->pri.vid = 0xFFFF;
1607 qp->alt.vid = 0xFFFF;
1608
1609 case IB_QPT_UD:
1610 {
1611 err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1612 init_attr, udata, 0, &qp);
1613 if (err) {
1614 kfree(qp);
1615 return ERR_PTR(err);
1616 }
1617
1618 qp->ibqp.qp_num = qp->mqp.qpn;
1619 qp->xrcdn = xrcdn;
1620
1621 break;
1622 }
1623 case IB_QPT_SMI:
1624 case IB_QPT_GSI:
1625 {
1626 int sqpn;
1627
1628
1629 if (udata)
1630 return ERR_PTR(-EINVAL);
1631 if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
1632 int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1633 1, 1, &sqpn, 0,
1634 MLX4_RES_USAGE_DRIVER);
1635
1636 if (res)
1637 return ERR_PTR(res);
1638 } else {
1639 sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
1640 }
1641
1642 err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
1643 init_attr, udata, sqpn, &qp);
1644 if (err)
1645 return ERR_PTR(err);
1646
1647 qp->port = init_attr->port_num;
1648 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 :
1649 init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI ? sqpn : 1;
1650 break;
1651 }
1652 default:
1653
1654 return ERR_PTR(-EINVAL);
1655 }
1656
1657 return &qp->ibqp;
1658}
1659
1660struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
1661 struct ib_qp_init_attr *init_attr,
1662 struct ib_udata *udata) {
1663 struct ib_device *device = pd ? pd->device : init_attr->xrcd->device;
1664 struct ib_qp *ibqp;
1665 struct mlx4_ib_dev *dev = to_mdev(device);
1666
1667 ibqp = _mlx4_ib_create_qp(pd, init_attr, udata);
1668
1669 if (!IS_ERR(ibqp) &&
1670 (init_attr->qp_type == IB_QPT_GSI) &&
1671 !(init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI)) {
1672 struct mlx4_ib_sqp *sqp = to_msqp((to_mqp(ibqp)));
1673 int is_eth = rdma_cap_eth_ah(&dev->ib_dev, init_attr->port_num);
1674
1675 if (is_eth &&
1676 dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ROCE_V1_V2) {
1677 init_attr->create_flags |= MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1678 sqp->roce_v2_gsi = ib_create_qp(pd, init_attr);
1679
1680 if (IS_ERR(sqp->roce_v2_gsi)) {
1681 pr_err("Failed to create GSI QP for RoCEv2 (%ld)\n", PTR_ERR(sqp->roce_v2_gsi));
1682 sqp->roce_v2_gsi = NULL;
1683 } else {
1684 sqp = to_msqp(to_mqp(sqp->roce_v2_gsi));
1685 sqp->qp.flags |= MLX4_IB_ROCE_V2_GSI_QP;
1686 }
1687
1688 init_attr->create_flags &= ~MLX4_IB_QP_CREATE_ROCE_V2_GSI;
1689 }
1690 }
1691 return ibqp;
1692}
1693
1694static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
1695{
1696 struct mlx4_ib_dev *dev = to_mdev(qp->device);
1697 struct mlx4_ib_qp *mqp = to_mqp(qp);
1698
1699 if (is_qp0(dev, mqp))
1700 mlx4_CLOSE_PORT(dev->dev, mqp->port);
1701
1702 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI &&
1703 dev->qp1_proxy[mqp->port - 1] == mqp) {
1704 mutex_lock(&dev->qp1_proxy_lock[mqp->port - 1]);
1705 dev->qp1_proxy[mqp->port - 1] = NULL;
1706 mutex_unlock(&dev->qp1_proxy_lock[mqp->port - 1]);
1707 }
1708
1709 if (mqp->counter_index)
1710 mlx4_ib_free_qp_counter(dev, mqp);
1711
1712 if (qp->rwq_ind_tbl) {
1713 destroy_qp_rss(dev, mqp);
1714 } else {
1715 struct mlx4_ib_pd *pd;
1716
1717 pd = get_pd(mqp);
1718 destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, !!pd->ibpd.uobject);
1719 }
1720
1721 if (is_sqp(dev, mqp))
1722 kfree(to_msqp(mqp));
1723 else
1724 kfree(mqp);
1725
1726 return 0;
1727}
1728
1729int mlx4_ib_destroy_qp(struct ib_qp *qp)
1730{
1731 struct mlx4_ib_qp *mqp = to_mqp(qp);
1732
1733 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
1734 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
1735
1736 if (sqp->roce_v2_gsi)
1737 ib_destroy_qp(sqp->roce_v2_gsi);
1738 }
1739
1740 return _mlx4_ib_destroy_qp(qp);
1741}
1742
1743static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
1744{
1745 switch (type) {
1746 case MLX4_IB_QPT_RC: return MLX4_QP_ST_RC;
1747 case MLX4_IB_QPT_UC: return MLX4_QP_ST_UC;
1748 case MLX4_IB_QPT_UD: return MLX4_QP_ST_UD;
1749 case MLX4_IB_QPT_XRC_INI:
1750 case MLX4_IB_QPT_XRC_TGT: return MLX4_QP_ST_XRC;
1751 case MLX4_IB_QPT_SMI:
1752 case MLX4_IB_QPT_GSI:
1753 case MLX4_IB_QPT_RAW_PACKET: return MLX4_QP_ST_MLX;
1754
1755 case MLX4_IB_QPT_PROXY_SMI_OWNER:
1756 case MLX4_IB_QPT_TUN_SMI_OWNER: return (mlx4_is_mfunc(dev->dev) ?
1757 MLX4_QP_ST_MLX : -1);
1758 case MLX4_IB_QPT_PROXY_SMI:
1759 case MLX4_IB_QPT_TUN_SMI:
1760 case MLX4_IB_QPT_PROXY_GSI:
1761 case MLX4_IB_QPT_TUN_GSI: return (mlx4_is_mfunc(dev->dev) ?
1762 MLX4_QP_ST_UD : -1);
1763 default: return -1;
1764 }
1765}
1766
1767static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
1768 int attr_mask)
1769{
1770 u8 dest_rd_atomic;
1771 u32 access_flags;
1772 u32 hw_access_flags = 0;
1773
1774 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1775 dest_rd_atomic = attr->max_dest_rd_atomic;
1776 else
1777 dest_rd_atomic = qp->resp_depth;
1778
1779 if (attr_mask & IB_QP_ACCESS_FLAGS)
1780 access_flags = attr->qp_access_flags;
1781 else
1782 access_flags = qp->atomic_rd_en;
1783
1784 if (!dest_rd_atomic)
1785 access_flags &= IB_ACCESS_REMOTE_WRITE;
1786
1787 if (access_flags & IB_ACCESS_REMOTE_READ)
1788 hw_access_flags |= MLX4_QP_BIT_RRE;
1789 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
1790 hw_access_flags |= MLX4_QP_BIT_RAE;
1791 if (access_flags & IB_ACCESS_REMOTE_WRITE)
1792 hw_access_flags |= MLX4_QP_BIT_RWE;
1793
1794 return cpu_to_be32(hw_access_flags);
1795}
1796
1797static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
1798 int attr_mask)
1799{
1800 if (attr_mask & IB_QP_PKEY_INDEX)
1801 sqp->pkey_index = attr->pkey_index;
1802 if (attr_mask & IB_QP_QKEY)
1803 sqp->qkey = attr->qkey;
1804 if (attr_mask & IB_QP_SQ_PSN)
1805 sqp->send_psn = attr->sq_psn;
1806}
1807
1808static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
1809{
1810 path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
1811}
1812
1813static int _mlx4_set_path(struct mlx4_ib_dev *dev,
1814 const struct rdma_ah_attr *ah,
1815 u64 smac, u16 vlan_tag, struct mlx4_qp_path *path,
1816 struct mlx4_roce_smac_vlan_info *smac_info, u8 port)
1817{
1818 int vidx;
1819 int smac_index;
1820 int err;
1821
1822 path->grh_mylmc = rdma_ah_get_path_bits(ah) & 0x7f;
1823 path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
1824 if (rdma_ah_get_static_rate(ah)) {
1825 path->static_rate = rdma_ah_get_static_rate(ah) +
1826 MLX4_STAT_RATE_OFFSET;
1827 while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
1828 !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
1829 --path->static_rate;
1830 } else
1831 path->static_rate = 0;
1832
1833 if (rdma_ah_get_ah_flags(ah) & IB_AH_GRH) {
1834 const struct ib_global_route *grh = rdma_ah_read_grh(ah);
1835 int real_sgid_index =
1836 mlx4_ib_gid_index_to_real_index(dev, port,
1837 grh->sgid_index);
1838
1839 if (real_sgid_index >= dev->dev->caps.gid_table_len[port]) {
1840 pr_err("sgid_index (%u) too large. max is %d\n",
1841 real_sgid_index, dev->dev->caps.gid_table_len[port] - 1);
1842 return -1;
1843 }
1844
1845 path->grh_mylmc |= 1 << 7;
1846 path->mgid_index = real_sgid_index;
1847 path->hop_limit = grh->hop_limit;
1848 path->tclass_flowlabel =
1849 cpu_to_be32((grh->traffic_class << 20) |
1850 (grh->flow_label));
1851 memcpy(path->rgid, grh->dgid.raw, 16);
1852 }
1853
1854 if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
1855 if (!(rdma_ah_get_ah_flags(ah) & IB_AH_GRH))
1856 return -1;
1857
1858 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1859 ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 7) << 3);
1860
1861 path->feup |= MLX4_FEUP_FORCE_ETH_UP;
1862 if (vlan_tag < 0x1000) {
1863 if (smac_info->vid < 0x1000) {
1864
1865 if (smac_info->vid != vlan_tag) {
1866
1867 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1868 if (err)
1869 return err;
1870 smac_info->candidate_vid = vlan_tag;
1871 smac_info->candidate_vlan_index = vidx;
1872 smac_info->candidate_vlan_port = port;
1873 smac_info->update_vid = 1;
1874 path->vlan_index = vidx;
1875 } else {
1876 path->vlan_index = smac_info->vlan_index;
1877 }
1878 } else {
1879
1880 err = mlx4_register_vlan(dev->dev, port, vlan_tag, &vidx);
1881 if (err)
1882 return err;
1883 smac_info->candidate_vid = vlan_tag;
1884 smac_info->candidate_vlan_index = vidx;
1885 smac_info->candidate_vlan_port = port;
1886 smac_info->update_vid = 1;
1887 path->vlan_index = vidx;
1888 }
1889 path->feup |= MLX4_FVL_FORCE_ETH_VLAN;
1890 path->fl = 1 << 6;
1891 } else {
1892
1893 if (smac_info->vid < 0x1000) {
1894 smac_info->candidate_vid = 0xFFFF;
1895 smac_info->update_vid = 1;
1896 }
1897 }
1898
1899
1900
1901
1902
1903
1904 if ((!smac_info->smac && !smac_info->smac_port) ||
1905 smac_info->smac != smac) {
1906
1907 smac_index = mlx4_register_mac(dev->dev, port, smac);
1908 if (smac_index >= 0) {
1909 smac_info->candidate_smac_index = smac_index;
1910 smac_info->candidate_smac = smac;
1911 smac_info->candidate_smac_port = port;
1912 } else {
1913 return -EINVAL;
1914 }
1915 } else {
1916 smac_index = smac_info->smac_index;
1917 }
1918 memcpy(path->dmac, ah->roce.dmac, 6);
1919 path->ackto = MLX4_IB_LINK_TYPE_ETH;
1920
1921 path->grh_mylmc = (u8) (smac_index) | 0x80;
1922 } else {
1923 path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
1924 ((port - 1) << 6) | ((rdma_ah_get_sl(ah) & 0xf) << 2);
1925 }
1926
1927 return 0;
1928}
1929
1930static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_qp_attr *qp,
1931 enum ib_qp_attr_mask qp_attr_mask,
1932 struct mlx4_ib_qp *mqp,
1933 struct mlx4_qp_path *path, u8 port,
1934 u16 vlan_id, u8 *smac)
1935{
1936 return _mlx4_set_path(dev, &qp->ah_attr,
1937 mlx4_mac_to_u64(smac),
1938 vlan_id,
1939 path, &mqp->pri, port);
1940}
1941
1942static int mlx4_set_alt_path(struct mlx4_ib_dev *dev,
1943 const struct ib_qp_attr *qp,
1944 enum ib_qp_attr_mask qp_attr_mask,
1945 struct mlx4_ib_qp *mqp,
1946 struct mlx4_qp_path *path, u8 port)
1947{
1948 return _mlx4_set_path(dev, &qp->alt_ah_attr,
1949 0,
1950 0xffff,
1951 path, &mqp->alt, port);
1952}
1953
1954static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1955{
1956 struct mlx4_ib_gid_entry *ge, *tmp;
1957
1958 list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1959 if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
1960 ge->added = 1;
1961 ge->port = qp->port;
1962 }
1963 }
1964}
1965
1966static int handle_eth_ud_smac_index(struct mlx4_ib_dev *dev,
1967 struct mlx4_ib_qp *qp,
1968 struct mlx4_qp_context *context)
1969{
1970 u64 u64_mac;
1971 int smac_index;
1972
1973 u64_mac = atomic64_read(&dev->iboe.mac[qp->port - 1]);
1974
1975 context->pri_path.sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE | ((qp->port - 1) << 6);
1976 if (!qp->pri.smac && !qp->pri.smac_port) {
1977 smac_index = mlx4_register_mac(dev->dev, qp->port, u64_mac);
1978 if (smac_index >= 0) {
1979 qp->pri.candidate_smac_index = smac_index;
1980 qp->pri.candidate_smac = u64_mac;
1981 qp->pri.candidate_smac_port = qp->port;
1982 context->pri_path.grh_mylmc = 0x80 | (u8) smac_index;
1983 } else {
1984 return -ENOENT;
1985 }
1986 }
1987 return 0;
1988}
1989
1990static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
1991{
1992 struct counter_index *new_counter_index;
1993 int err;
1994 u32 tmp_idx;
1995
1996 if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) !=
1997 IB_LINK_LAYER_ETHERNET ||
1998 !(qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK) ||
1999 !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
2000 return 0;
2001
2002 err = mlx4_counter_alloc(dev->dev, &tmp_idx, MLX4_RES_USAGE_DRIVER);
2003 if (err)
2004 return err;
2005
2006 new_counter_index = kmalloc(sizeof(*new_counter_index), GFP_KERNEL);
2007 if (!new_counter_index) {
2008 mlx4_counter_free(dev->dev, tmp_idx);
2009 return -ENOMEM;
2010 }
2011
2012 new_counter_index->index = tmp_idx;
2013 new_counter_index->allocated = 1;
2014 qp->counter_index = new_counter_index;
2015
2016 mutex_lock(&dev->counters_table[qp->port - 1].mutex);
2017 list_add_tail(&new_counter_index->list,
2018 &dev->counters_table[qp->port - 1].counters_list);
2019 mutex_unlock(&dev->counters_table[qp->port - 1].mutex);
2020
2021 return 0;
2022}
2023
2024enum {
2025 MLX4_QPC_ROCE_MODE_1 = 0,
2026 MLX4_QPC_ROCE_MODE_2 = 2,
2027 MLX4_QPC_ROCE_MODE_UNDEFINED = 0xff
2028};
2029
2030static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
2031{
2032 switch (gid_type) {
2033 case IB_GID_TYPE_ROCE:
2034 return MLX4_QPC_ROCE_MODE_1;
2035 case IB_GID_TYPE_ROCE_UDP_ENCAP:
2036 return MLX4_QPC_ROCE_MODE_2;
2037 default:
2038 return MLX4_QPC_ROCE_MODE_UNDEFINED;
2039 }
2040}
2041
2042
2043
2044
2045
2046static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num)
2047{
2048 int err = 0;
2049 int i;
2050
2051 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
2052 struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
2053 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2054
2055 mutex_lock(&wq->mutex);
2056
2057
2058
2059
2060
2061
2062
2063 if ((wq->rss_usecnt > 0) && (wq->port != port_num)) {
2064 err = -EINVAL;
2065 mutex_unlock(&wq->mutex);
2066 break;
2067 }
2068 wq->port = port_num;
2069 if ((wq->rss_usecnt == 0) && (ibwq->state == IB_WQS_RDY)) {
2070 err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY);
2071 if (err) {
2072 mutex_unlock(&wq->mutex);
2073 break;
2074 }
2075 }
2076 wq->rss_usecnt++;
2077
2078 mutex_unlock(&wq->mutex);
2079 }
2080
2081 if (i && err) {
2082 int j;
2083
2084 for (j = (i - 1); j >= 0; j--) {
2085 struct ib_wq *ibwq = ind_tbl->ind_tbl[j];
2086 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2087
2088 mutex_lock(&wq->mutex);
2089
2090 if ((wq->rss_usecnt == 1) &&
2091 (ibwq->state == IB_WQS_RDY))
2092 if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
2093 pr_warn("failed to reverse WQN=0x%06x\n",
2094 ibwq->wq_num);
2095 wq->rss_usecnt--;
2096
2097 mutex_unlock(&wq->mutex);
2098 }
2099 }
2100
2101 return err;
2102}
2103
2104static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl)
2105{
2106 int i;
2107
2108 for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
2109 struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
2110 struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
2111
2112 mutex_lock(&wq->mutex);
2113
2114 if ((wq->rss_usecnt == 1) && (ibwq->state == IB_WQS_RDY))
2115 if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
2116 pr_warn("failed to reverse WQN=%x\n",
2117 ibwq->wq_num);
2118 wq->rss_usecnt--;
2119
2120 mutex_unlock(&wq->mutex);
2121 }
2122}
2123
2124static void fill_qp_rss_context(struct mlx4_qp_context *context,
2125 struct mlx4_ib_qp *qp)
2126{
2127 struct mlx4_rss_context *rss_context;
2128
2129 rss_context = (void *)context + offsetof(struct mlx4_qp_context,
2130 pri_path) + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
2131
2132 rss_context->base_qpn = cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz);
2133 rss_context->default_qpn =
2134 cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz & 0xffffff);
2135 if (qp->rss_ctx->flags & (MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6))
2136 rss_context->base_qpn_udp = rss_context->default_qpn;
2137 rss_context->flags = qp->rss_ctx->flags;
2138
2139 rss_context->hash_fn = MLX4_RSS_HASH_TOP;
2140
2141 memcpy(rss_context->rss_key, qp->rss_ctx->rss_key,
2142 MLX4_EN_RSS_KEY_SIZE);
2143}
2144
2145static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
2146 const struct ib_qp_attr *attr, int attr_mask,
2147 enum ib_qp_state cur_state, enum ib_qp_state new_state)
2148{
2149 struct ib_uobject *ibuobject;
2150 struct ib_srq *ibsrq;
2151 struct ib_rwq_ind_table *rwq_ind_tbl;
2152 enum ib_qp_type qp_type;
2153 struct mlx4_ib_dev *dev;
2154 struct mlx4_ib_qp *qp;
2155 struct mlx4_ib_pd *pd;
2156 struct mlx4_ib_cq *send_cq, *recv_cq;
2157 struct mlx4_qp_context *context;
2158 enum mlx4_qp_optpar optpar = 0;
2159 int sqd_event;
2160 int steer_qp = 0;
2161 int err = -EINVAL;
2162 int counter_index;
2163
2164 if (src_type == MLX4_IB_RWQ_SRC) {
2165 struct ib_wq *ibwq;
2166
2167 ibwq = (struct ib_wq *)src;
2168 ibuobject = ibwq->uobject;
2169 ibsrq = NULL;
2170 rwq_ind_tbl = NULL;
2171 qp_type = IB_QPT_RAW_PACKET;
2172 qp = to_mqp((struct ib_qp *)ibwq);
2173 dev = to_mdev(ibwq->device);
2174 pd = to_mpd(ibwq->pd);
2175 } else {
2176 struct ib_qp *ibqp;
2177
2178 ibqp = (struct ib_qp *)src;
2179 ibuobject = ibqp->uobject;
2180 ibsrq = ibqp->srq;
2181 rwq_ind_tbl = ibqp->rwq_ind_tbl;
2182 qp_type = ibqp->qp_type;
2183 qp = to_mqp(ibqp);
2184 dev = to_mdev(ibqp->device);
2185 pd = get_pd(qp);
2186 }
2187
2188
2189 if (attr_mask & IB_QP_ALT_PATH &&
2190 rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2191 IB_LINK_LAYER_ETHERNET)
2192 return -ENOTSUPP;
2193
2194 context = kzalloc(sizeof *context, GFP_KERNEL);
2195 if (!context)
2196 return -ENOMEM;
2197
2198 context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
2199 (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16));
2200
2201 if (!(attr_mask & IB_QP_PATH_MIG_STATE))
2202 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2203 else {
2204 optpar |= MLX4_QP_OPTPAR_PM_STATE;
2205 switch (attr->path_mig_state) {
2206 case IB_MIG_MIGRATED:
2207 context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
2208 break;
2209 case IB_MIG_REARM:
2210 context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
2211 break;
2212 case IB_MIG_ARMED:
2213 context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
2214 break;
2215 }
2216 }
2217
2218 if (qp->inl_recv_sz)
2219 context->param3 |= cpu_to_be32(1 << 25);
2220
2221 if (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI)
2222 context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
2223 else if (qp_type == IB_QPT_RAW_PACKET)
2224 context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
2225 else if (qp_type == IB_QPT_UD) {
2226 if (qp->flags & MLX4_IB_QP_LSO)
2227 context->mtu_msgmax = (IB_MTU_4096 << 5) |
2228 ilog2(dev->dev->caps.max_gso_sz);
2229 else
2230 context->mtu_msgmax = (IB_MTU_4096 << 5) | 13;
2231 } else if (attr_mask & IB_QP_PATH_MTU) {
2232 if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
2233 pr_err("path MTU (%u) is invalid\n",
2234 attr->path_mtu);
2235 goto out;
2236 }
2237 context->mtu_msgmax = (attr->path_mtu << 5) |
2238 ilog2(dev->dev->caps.max_msg_sz);
2239 }
2240
2241 if (!rwq_ind_tbl) {
2242 if (qp->rq.wqe_cnt)
2243 context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
2244 context->rq_size_stride |= qp->rq.wqe_shift - 4;
2245 }
2246
2247 if (qp->sq.wqe_cnt)
2248 context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
2249 context->sq_size_stride |= qp->sq.wqe_shift - 4;
2250
2251 if (new_state == IB_QPS_RESET && qp->counter_index)
2252 mlx4_ib_free_qp_counter(dev, qp);
2253
2254 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2255 context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
2256 context->xrcd = cpu_to_be32((u32) qp->xrcdn);
2257 if (qp_type == IB_QPT_RAW_PACKET)
2258 context->param3 |= cpu_to_be32(1 << 30);
2259 }
2260
2261 if (ibuobject)
2262 context->usr_page = cpu_to_be32(
2263 mlx4_to_hw_uar_index(dev->dev,
2264 to_mucontext(ibuobject->context)
2265 ->uar.index));
2266 else
2267 context->usr_page = cpu_to_be32(
2268 mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
2269
2270 if (attr_mask & IB_QP_DEST_QPN)
2271 context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
2272
2273 if (attr_mask & IB_QP_PORT) {
2274 if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
2275 !(attr_mask & IB_QP_AV)) {
2276 mlx4_set_sched(&context->pri_path, attr->port_num);
2277 optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
2278 }
2279 }
2280
2281 if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2282 err = create_qp_lb_counter(dev, qp);
2283 if (err)
2284 goto out;
2285
2286 counter_index =
2287 dev->counters_table[qp->port - 1].default_counter;
2288 if (qp->counter_index)
2289 counter_index = qp->counter_index->index;
2290
2291 if (counter_index != -1) {
2292 context->pri_path.counter_index = counter_index;
2293 optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX;
2294 if (qp->counter_index) {
2295 context->pri_path.fl |=
2296 MLX4_FL_ETH_SRC_CHECK_MC_LB;
2297 context->pri_path.vlan_control |=
2298 MLX4_CTRL_ETH_SRC_CHECK_IF_COUNTER;
2299 }
2300 } else
2301 context->pri_path.counter_index =
2302 MLX4_SINK_COUNTER_INDEX(dev->dev);
2303
2304 if (qp->flags & MLX4_IB_QP_NETIF) {
2305 mlx4_ib_steer_qp_reg(dev, qp, 1);
2306 steer_qp = 1;
2307 }
2308
2309 if (qp_type == IB_QPT_GSI) {
2310 enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ?
2311 IB_GID_TYPE_ROCE_UDP_ENCAP : IB_GID_TYPE_ROCE;
2312 u8 qpc_roce_mode = gid_type_to_qpc(gid_type);
2313
2314 context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2315 }
2316 }
2317
2318 if (attr_mask & IB_QP_PKEY_INDEX) {
2319 if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2320 context->pri_path.disable_pkey_check = 0x40;
2321 context->pri_path.pkey_index = attr->pkey_index;
2322 optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
2323 }
2324
2325 if (attr_mask & IB_QP_AV) {
2326 u8 port_num = mlx4_is_bonded(dev->dev) ? 1 :
2327 attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2328 union ib_gid gid;
2329 struct ib_gid_attr gid_attr = {.gid_type = IB_GID_TYPE_IB};
2330 u16 vlan = 0xffff;
2331 u8 smac[ETH_ALEN];
2332 int status = 0;
2333 int is_eth =
2334 rdma_cap_eth_ah(&dev->ib_dev, port_num) &&
2335 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
2336
2337 if (is_eth) {
2338 int index =
2339 rdma_ah_read_grh(&attr->ah_attr)->sgid_index;
2340
2341 status = ib_get_cached_gid(&dev->ib_dev, port_num,
2342 index, &gid, &gid_attr);
2343 if (!status && !memcmp(&gid, &zgid, sizeof(gid)))
2344 status = -ENOENT;
2345 if (!status && gid_attr.ndev) {
2346 vlan = rdma_vlan_dev_vlan_id(gid_attr.ndev);
2347 memcpy(smac, gid_attr.ndev->dev_addr, ETH_ALEN);
2348 dev_put(gid_attr.ndev);
2349 }
2350 }
2351 if (status)
2352 goto out;
2353
2354 if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
2355 port_num, vlan, smac))
2356 goto out;
2357
2358 optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
2359 MLX4_QP_OPTPAR_SCHED_QUEUE);
2360
2361 if (is_eth &&
2362 (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)) {
2363 u8 qpc_roce_mode = gid_type_to_qpc(gid_attr.gid_type);
2364
2365 if (qpc_roce_mode == MLX4_QPC_ROCE_MODE_UNDEFINED) {
2366 err = -EINVAL;
2367 goto out;
2368 }
2369 context->rlkey_roce_mode |= (qpc_roce_mode << 6);
2370 }
2371
2372 }
2373
2374 if (attr_mask & IB_QP_TIMEOUT) {
2375 context->pri_path.ackto |= attr->timeout << 3;
2376 optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
2377 }
2378
2379 if (attr_mask & IB_QP_ALT_PATH) {
2380 if (attr->alt_port_num == 0 ||
2381 attr->alt_port_num > dev->dev->caps.num_ports)
2382 goto out;
2383
2384 if (attr->alt_pkey_index >=
2385 dev->dev->caps.pkey_table_len[attr->alt_port_num])
2386 goto out;
2387
2388 if (mlx4_set_alt_path(dev, attr, attr_mask, qp,
2389 &context->alt_path,
2390 attr->alt_port_num))
2391 goto out;
2392
2393 context->alt_path.pkey_index = attr->alt_pkey_index;
2394 context->alt_path.ackto = attr->alt_timeout << 3;
2395 optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
2396 }
2397
2398 context->pd = cpu_to_be32(pd->pdn);
2399
2400 if (!rwq_ind_tbl) {
2401 context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
2402 get_cqs(qp, src_type, &send_cq, &recv_cq);
2403 } else {
2404 send_cq = to_mcq(rwq_ind_tbl->ind_tbl[0]->cq);
2405 recv_cq = send_cq;
2406 }
2407 context->cqn_send = cpu_to_be32(send_cq->mcq.cqn);
2408 context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
2409
2410
2411 if (!ibuobject)
2412 context->params1 |= cpu_to_be32(1 << 11);
2413
2414 if (attr_mask & IB_QP_RNR_RETRY) {
2415 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
2416 optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
2417 }
2418
2419 if (attr_mask & IB_QP_RETRY_CNT) {
2420 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
2421 optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
2422 }
2423
2424 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2425 if (attr->max_rd_atomic)
2426 context->params1 |=
2427 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
2428 optpar |= MLX4_QP_OPTPAR_SRA_MAX;
2429 }
2430
2431 if (attr_mask & IB_QP_SQ_PSN)
2432 context->next_send_psn = cpu_to_be32(attr->sq_psn);
2433
2434 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2435 if (attr->max_dest_rd_atomic)
2436 context->params2 |=
2437 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
2438 optpar |= MLX4_QP_OPTPAR_RRA_MAX;
2439 }
2440
2441 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
2442 context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
2443 optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
2444 }
2445
2446 if (ibsrq)
2447 context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
2448
2449 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2450 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
2451 optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
2452 }
2453 if (attr_mask & IB_QP_RQ_PSN)
2454 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
2455
2456
2457 if (attr_mask & IB_QP_QKEY) {
2458 if (qp->mlx4_ib_qp_type &
2459 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))
2460 context->qkey = cpu_to_be32(IB_QP_SET_QKEY);
2461 else {
2462 if (mlx4_is_mfunc(dev->dev) &&
2463 !(qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV) &&
2464 (attr->qkey & MLX4_RESERVED_QKEY_MASK) ==
2465 MLX4_RESERVED_QKEY_BASE) {
2466 pr_err("Cannot use reserved QKEY"
2467 " 0x%x (range 0xffff0000..0xffffffff"
2468 " is reserved)\n", attr->qkey);
2469 err = -EINVAL;
2470 goto out;
2471 }
2472 context->qkey = cpu_to_be32(attr->qkey);
2473 }
2474 optpar |= MLX4_QP_OPTPAR_Q_KEY;
2475 }
2476
2477 if (ibsrq)
2478 context->srqn = cpu_to_be32(1 << 24 |
2479 to_msrq(ibsrq)->msrq.srqn);
2480
2481 if (qp->rq.wqe_cnt &&
2482 cur_state == IB_QPS_RESET &&
2483 new_state == IB_QPS_INIT)
2484 context->db_rec_addr = cpu_to_be64(qp->db.dma);
2485
2486 if (cur_state == IB_QPS_INIT &&
2487 new_state == IB_QPS_RTR &&
2488 (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI ||
2489 qp_type == IB_QPT_UD || qp_type == IB_QPT_RAW_PACKET)) {
2490 context->pri_path.sched_queue = (qp->port - 1) << 6;
2491 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
2492 qp->mlx4_ib_qp_type &
2493 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER)) {
2494 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
2495 if (qp->mlx4_ib_qp_type != MLX4_IB_QPT_SMI)
2496 context->pri_path.fl = 0x80;
2497 } else {
2498 if (qp->mlx4_ib_qp_type & MLX4_IB_QPT_ANY_SRIOV)
2499 context->pri_path.fl = 0x80;
2500 context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
2501 }
2502 if (rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
2503 IB_LINK_LAYER_ETHERNET) {
2504 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI ||
2505 qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI)
2506 context->pri_path.feup = 1 << 7;
2507
2508 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_UD ||
2509 qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI ||
2510 qp->mlx4_ib_qp_type == MLX4_IB_QPT_TUN_GSI) {
2511 err = handle_eth_ud_smac_index(dev, qp, context);
2512 if (err) {
2513 err = -EINVAL;
2514 goto out;
2515 }
2516 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_GSI)
2517 dev->qp1_proxy[qp->port - 1] = qp;
2518 }
2519 }
2520 }
2521
2522 if (qp_type == IB_QPT_RAW_PACKET) {
2523 context->pri_path.ackto = (context->pri_path.ackto & 0xf8) |
2524 MLX4_IB_LINK_TYPE_ETH;
2525 if (dev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
2526
2527 if (!rwq_ind_tbl)
2528 context->srqn = cpu_to_be32(7 << 28);
2529 }
2530 }
2531
2532 if (qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
2533 int is_eth = rdma_port_get_link_layer(
2534 &dev->ib_dev, qp->port) ==
2535 IB_LINK_LAYER_ETHERNET;
2536 if (is_eth) {
2537 context->pri_path.ackto = MLX4_IB_LINK_TYPE_ETH;
2538 optpar |= MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH;
2539 }
2540 }
2541
2542 if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD &&
2543 attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
2544 sqd_event = 1;
2545 else
2546 sqd_event = 0;
2547
2548 if (!ibuobject &&
2549 cur_state == IB_QPS_RESET &&
2550 new_state == IB_QPS_INIT)
2551 context->rlkey_roce_mode |= (1 << 4);
2552
2553
2554
2555
2556
2557
2558
2559 if (!ibuobject &&
2560 cur_state == IB_QPS_RESET &&
2561 new_state == IB_QPS_INIT) {
2562 struct mlx4_wqe_ctrl_seg *ctrl;
2563 int i;
2564
2565 for (i = 0; i < qp->sq.wqe_cnt; ++i) {
2566 ctrl = get_send_wqe(qp, i);
2567 ctrl->owner_opcode = cpu_to_be32(1 << 31);
2568 if (qp->sq_max_wqes_per_wr == 1)
2569 ctrl->qpn_vlan.fence_size =
2570 1 << (qp->sq.wqe_shift - 4);
2571
2572 stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
2573 }
2574 }
2575
2576 if (rwq_ind_tbl &&
2577 cur_state == IB_QPS_RESET &&
2578 new_state == IB_QPS_INIT) {
2579 fill_qp_rss_context(context, qp);
2580 context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET);
2581 }
2582
2583 err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
2584 to_mlx4_state(new_state), context, optpar,
2585 sqd_event, &qp->mqp);
2586 if (err)
2587 goto out;
2588
2589 qp->state = new_state;
2590
2591 if (attr_mask & IB_QP_ACCESS_FLAGS)
2592 qp->atomic_rd_en = attr->qp_access_flags;
2593 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2594 qp->resp_depth = attr->max_dest_rd_atomic;
2595 if (attr_mask & IB_QP_PORT) {
2596 qp->port = attr->port_num;
2597 update_mcg_macs(dev, qp);
2598 }
2599 if (attr_mask & IB_QP_ALT_PATH)
2600 qp->alt_port = attr->alt_port_num;
2601
2602 if (is_sqp(dev, qp))
2603 store_sqp_attrs(to_msqp(qp), attr, attr_mask);
2604
2605
2606
2607
2608
2609 if (is_qp0(dev, qp)) {
2610 if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
2611 if (mlx4_INIT_PORT(dev->dev, qp->port))
2612 pr_warn("INIT_PORT failed for port %d\n",
2613 qp->port);
2614
2615 if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
2616 (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
2617 mlx4_CLOSE_PORT(dev->dev, qp->port);
2618 }
2619
2620
2621
2622
2623
2624 if (new_state == IB_QPS_RESET) {
2625 if (!ibuobject) {
2626 mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
2627 ibsrq ? to_msrq(ibsrq) : NULL);
2628 if (send_cq != recv_cq)
2629 mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
2630
2631 qp->rq.head = 0;
2632 qp->rq.tail = 0;
2633 qp->sq.head = 0;
2634 qp->sq.tail = 0;
2635 qp->sq_next_wqe = 0;
2636 if (qp->rq.wqe_cnt)
2637 *qp->db.db = 0;
2638
2639 if (qp->flags & MLX4_IB_QP_NETIF)
2640 mlx4_ib_steer_qp_reg(dev, qp, 0);
2641 }
2642 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port)) {
2643 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2644 qp->pri.smac = 0;
2645 qp->pri.smac_port = 0;
2646 }
2647 if (qp->alt.smac) {
2648 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2649 qp->alt.smac = 0;
2650 }
2651 if (qp->pri.vid < 0x1000) {
2652 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port, qp->pri.vid);
2653 qp->pri.vid = 0xFFFF;
2654 qp->pri.candidate_vid = 0xFFFF;
2655 qp->pri.update_vid = 0;
2656 }
2657
2658 if (qp->alt.vid < 0x1000) {
2659 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port, qp->alt.vid);
2660 qp->alt.vid = 0xFFFF;
2661 qp->alt.candidate_vid = 0xFFFF;
2662 qp->alt.update_vid = 0;
2663 }
2664 }
2665out:
2666 if (err && qp->counter_index)
2667 mlx4_ib_free_qp_counter(dev, qp);
2668 if (err && steer_qp)
2669 mlx4_ib_steer_qp_reg(dev, qp, 0);
2670 kfree(context);
2671 if (qp->pri.candidate_smac ||
2672 (!qp->pri.candidate_smac && qp->pri.candidate_smac_port)) {
2673 if (err) {
2674 mlx4_unregister_mac(dev->dev, qp->pri.candidate_smac_port, qp->pri.candidate_smac);
2675 } else {
2676 if (qp->pri.smac || (!qp->pri.smac && qp->pri.smac_port))
2677 mlx4_unregister_mac(dev->dev, qp->pri.smac_port, qp->pri.smac);
2678 qp->pri.smac = qp->pri.candidate_smac;
2679 qp->pri.smac_index = qp->pri.candidate_smac_index;
2680 qp->pri.smac_port = qp->pri.candidate_smac_port;
2681 }
2682 qp->pri.candidate_smac = 0;
2683 qp->pri.candidate_smac_index = 0;
2684 qp->pri.candidate_smac_port = 0;
2685 }
2686 if (qp->alt.candidate_smac) {
2687 if (err) {
2688 mlx4_unregister_mac(dev->dev, qp->alt.candidate_smac_port, qp->alt.candidate_smac);
2689 } else {
2690 if (qp->alt.smac)
2691 mlx4_unregister_mac(dev->dev, qp->alt.smac_port, qp->alt.smac);
2692 qp->alt.smac = qp->alt.candidate_smac;
2693 qp->alt.smac_index = qp->alt.candidate_smac_index;
2694 qp->alt.smac_port = qp->alt.candidate_smac_port;
2695 }
2696 qp->alt.candidate_smac = 0;
2697 qp->alt.candidate_smac_index = 0;
2698 qp->alt.candidate_smac_port = 0;
2699 }
2700
2701 if (qp->pri.update_vid) {
2702 if (err) {
2703 if (qp->pri.candidate_vid < 0x1000)
2704 mlx4_unregister_vlan(dev->dev, qp->pri.candidate_vlan_port,
2705 qp->pri.candidate_vid);
2706 } else {
2707 if (qp->pri.vid < 0x1000)
2708 mlx4_unregister_vlan(dev->dev, qp->pri.vlan_port,
2709 qp->pri.vid);
2710 qp->pri.vid = qp->pri.candidate_vid;
2711 qp->pri.vlan_port = qp->pri.candidate_vlan_port;
2712 qp->pri.vlan_index = qp->pri.candidate_vlan_index;
2713 }
2714 qp->pri.candidate_vid = 0xFFFF;
2715 qp->pri.update_vid = 0;
2716 }
2717
2718 if (qp->alt.update_vid) {
2719 if (err) {
2720 if (qp->alt.candidate_vid < 0x1000)
2721 mlx4_unregister_vlan(dev->dev, qp->alt.candidate_vlan_port,
2722 qp->alt.candidate_vid);
2723 } else {
2724 if (qp->alt.vid < 0x1000)
2725 mlx4_unregister_vlan(dev->dev, qp->alt.vlan_port,
2726 qp->alt.vid);
2727 qp->alt.vid = qp->alt.candidate_vid;
2728 qp->alt.vlan_port = qp->alt.candidate_vlan_port;
2729 qp->alt.vlan_index = qp->alt.candidate_vlan_index;
2730 }
2731 qp->alt.candidate_vid = 0xFFFF;
2732 qp->alt.update_vid = 0;
2733 }
2734
2735 return err;
2736}
2737
2738enum {
2739 MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK = (IB_QP_STATE |
2740 IB_QP_PORT),
2741};
2742
2743static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2744 int attr_mask, struct ib_udata *udata)
2745{
2746 enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
2747 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2748 struct mlx4_ib_qp *qp = to_mqp(ibqp);
2749 enum ib_qp_state cur_state, new_state;
2750 int err = -EINVAL;
2751 mutex_lock(&qp->mutex);
2752
2753 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
2754 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
2755
2756 if (cur_state != new_state || cur_state != IB_QPS_RESET) {
2757 int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2758 ll = rdma_port_get_link_layer(&dev->ib_dev, port);
2759 }
2760
2761 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
2762 attr_mask, ll)) {
2763 pr_debug("qpn 0x%x: invalid attribute mask specified "
2764 "for transition %d to %d. qp_type %d,"
2765 " attr_mask 0x%x\n",
2766 ibqp->qp_num, cur_state, new_state,
2767 ibqp->qp_type, attr_mask);
2768 goto out;
2769 }
2770
2771 if (ibqp->rwq_ind_tbl) {
2772 if (!(((cur_state == IB_QPS_RESET) &&
2773 (new_state == IB_QPS_INIT)) ||
2774 ((cur_state == IB_QPS_INIT) &&
2775 (new_state == IB_QPS_RTR)))) {
2776 pr_debug("qpn 0x%x: RSS QP unsupported transition %d to %d\n",
2777 ibqp->qp_num, cur_state, new_state);
2778
2779 err = -EOPNOTSUPP;
2780 goto out;
2781 }
2782
2783 if (attr_mask & ~MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK) {
2784 pr_debug("qpn 0x%x: RSS QP unsupported attribute mask 0x%x for transition %d to %d\n",
2785 ibqp->qp_num, attr_mask, cur_state, new_state);
2786
2787 err = -EOPNOTSUPP;
2788 goto out;
2789 }
2790 }
2791
2792 if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT)) {
2793 if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
2794 if ((ibqp->qp_type == IB_QPT_RC) ||
2795 (ibqp->qp_type == IB_QPT_UD) ||
2796 (ibqp->qp_type == IB_QPT_UC) ||
2797 (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
2798 (ibqp->qp_type == IB_QPT_XRC_INI)) {
2799 attr->port_num = mlx4_ib_bond_next_port(dev);
2800 }
2801 } else {
2802
2803
2804 attr_mask &= ~IB_QP_PORT;
2805 }
2806 }
2807
2808 if ((attr_mask & IB_QP_PORT) &&
2809 (attr->port_num == 0 || attr->port_num > dev->num_ports)) {
2810 pr_debug("qpn 0x%x: invalid port number (%d) specified "
2811 "for transition %d to %d. qp_type %d\n",
2812 ibqp->qp_num, attr->port_num, cur_state,
2813 new_state, ibqp->qp_type);
2814 goto out;
2815 }
2816
2817 if ((attr_mask & IB_QP_PORT) && (ibqp->qp_type == IB_QPT_RAW_PACKET) &&
2818 (rdma_port_get_link_layer(&dev->ib_dev, attr->port_num) !=
2819 IB_LINK_LAYER_ETHERNET))
2820 goto out;
2821
2822 if (attr_mask & IB_QP_PKEY_INDEX) {
2823 int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
2824 if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p]) {
2825 pr_debug("qpn 0x%x: invalid pkey index (%d) specified "
2826 "for transition %d to %d. qp_type %d\n",
2827 ibqp->qp_num, attr->pkey_index, cur_state,
2828 new_state, ibqp->qp_type);
2829 goto out;
2830 }
2831 }
2832
2833 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
2834 attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
2835 pr_debug("qpn 0x%x: max_rd_atomic (%d) too large. "
2836 "Transition %d to %d. qp_type %d\n",
2837 ibqp->qp_num, attr->max_rd_atomic, cur_state,
2838 new_state, ibqp->qp_type);
2839 goto out;
2840 }
2841
2842 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
2843 attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
2844 pr_debug("qpn 0x%x: max_dest_rd_atomic (%d) too large. "
2845 "Transition %d to %d. qp_type %d\n",
2846 ibqp->qp_num, attr->max_dest_rd_atomic, cur_state,
2847 new_state, ibqp->qp_type);
2848 goto out;
2849 }
2850
2851 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
2852 err = 0;
2853 goto out;
2854 }
2855
2856 if (ibqp->rwq_ind_tbl && (new_state == IB_QPS_INIT)) {
2857 err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num);
2858 if (err)
2859 goto out;
2860 }
2861
2862 err = __mlx4_ib_modify_qp(ibqp, MLX4_IB_QP_SRC, attr, attr_mask,
2863 cur_state, new_state);
2864
2865 if (ibqp->rwq_ind_tbl && err)
2866 bring_down_rss_rwqs(ibqp->rwq_ind_tbl);
2867
2868 if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
2869 attr->port_num = 1;
2870
2871out:
2872 mutex_unlock(&qp->mutex);
2873 return err;
2874}
2875
2876int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2877 int attr_mask, struct ib_udata *udata)
2878{
2879 struct mlx4_ib_qp *mqp = to_mqp(ibqp);
2880 int ret;
2881
2882 ret = _mlx4_ib_modify_qp(ibqp, attr, attr_mask, udata);
2883
2884 if (mqp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
2885 struct mlx4_ib_sqp *sqp = to_msqp(mqp);
2886 int err = 0;
2887
2888 if (sqp->roce_v2_gsi)
2889 err = ib_modify_qp(sqp->roce_v2_gsi, attr, attr_mask);
2890 if (err)
2891 pr_err("Failed to modify GSI QP for RoCEv2 (%d)\n",
2892 err);
2893 }
2894 return ret;
2895}
2896
2897static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
2898{
2899 int i;
2900 for (i = 0; i < dev->caps.num_ports; i++) {
2901 if (qpn == dev->caps.spec_qps[i].qp0_proxy ||
2902 qpn == dev->caps.spec_qps[i].qp0_tunnel) {
2903 *qkey = dev->caps.spec_qps[i].qp0_qkey;
2904 return 0;
2905 }
2906 }
2907 return -EINVAL;
2908}
2909
2910static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
2911 struct ib_ud_wr *wr,
2912 void *wqe, unsigned *mlx_seg_len)
2913{
2914 struct mlx4_ib_dev *mdev = to_mdev(sqp->qp.ibqp.device);
2915 struct ib_device *ib_dev = &mdev->ib_dev;
2916 struct mlx4_wqe_mlx_seg *mlx = wqe;
2917 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
2918 struct mlx4_ib_ah *ah = to_mah(wr->ah);
2919 u16 pkey;
2920 u32 qkey;
2921 int send_size;
2922 int header_size;
2923 int spc;
2924 int i;
2925
2926 if (wr->wr.opcode != IB_WR_SEND)
2927 return -EINVAL;
2928
2929 send_size = 0;
2930
2931 for (i = 0; i < wr->wr.num_sge; ++i)
2932 send_size += wr->wr.sg_list[i].length;
2933
2934
2935
2936 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER)
2937 send_size += sizeof (struct mlx4_ib_tunnel_header);
2938
2939 ib_ud_header_init(send_size, 1, 0, 0, 0, 0, 0, 0, &sqp->ud_header);
2940
2941 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_PROXY_SMI_OWNER) {
2942 sqp->ud_header.lrh.service_level =
2943 be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
2944 sqp->ud_header.lrh.destination_lid =
2945 cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2946 sqp->ud_header.lrh.source_lid =
2947 cpu_to_be16(ah->av.ib.g_slid & 0x7f);
2948 }
2949
2950 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
2951
2952
2953 mlx->flags |= cpu_to_be32(MLX4_WQE_MLX_VL15 | 0x1 | MLX4_WQE_MLX_SLR);
2954 mlx->rlid = sqp->ud_header.lrh.destination_lid;
2955
2956 sqp->ud_header.lrh.virtual_lane = 0;
2957 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
2958 ib_get_cached_pkey(ib_dev, sqp->qp.port, 0, &pkey);
2959 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
2960 if (sqp->qp.mlx4_ib_qp_type == MLX4_IB_QPT_TUN_SMI_OWNER)
2961 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
2962 else
2963 sqp->ud_header.bth.destination_qpn =
2964 cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel);
2965
2966 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
2967 if (mlx4_is_master(mdev->dev)) {
2968 if (mlx4_get_parav_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2969 return -EINVAL;
2970 } else {
2971 if (vf_get_qp0_qkey(mdev->dev, sqp->qp.mqp.qpn, &qkey))
2972 return -EINVAL;
2973 }
2974 sqp->ud_header.deth.qkey = cpu_to_be32(qkey);
2975 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.mqp.qpn);
2976
2977 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2978 sqp->ud_header.immediate_present = 0;
2979
2980 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
2981
2982
2983
2984
2985
2986
2987
2988 spc = MLX4_INLINE_ALIGN -
2989 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
2990 if (header_size <= spc) {
2991 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
2992 memcpy(inl + 1, sqp->header_buf, header_size);
2993 i = 1;
2994 } else {
2995 inl->byte_count = cpu_to_be32(1 << 31 | spc);
2996 memcpy(inl + 1, sqp->header_buf, spc);
2997
2998 inl = (void *) (inl + 1) + spc;
2999 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013 wmb();
3014 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
3015 i = 2;
3016 }
3017
3018 *mlx_seg_len =
3019 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
3020 return 0;
3021}
3022
3023static u8 sl_to_vl(struct mlx4_ib_dev *dev, u8 sl, int port_num)
3024{
3025 union sl2vl_tbl_to_u64 tmp_vltab;
3026 u8 vl;
3027
3028 if (sl > 15)
3029 return 0xf;
3030 tmp_vltab.sl64 = atomic64_read(&dev->sl2vl[port_num - 1]);
3031 vl = tmp_vltab.sl8[sl >> 1];
3032 if (sl & 1)
3033 vl &= 0x0f;
3034 else
3035 vl >>= 4;
3036 return vl;
3037}
3038
3039static int fill_gid_by_hw_index(struct mlx4_ib_dev *ibdev, u8 port_num,
3040 int index, union ib_gid *gid,
3041 enum ib_gid_type *gid_type)
3042{
3043 struct mlx4_ib_iboe *iboe = &ibdev->iboe;
3044 struct mlx4_port_gid_table *port_gid_table;
3045 unsigned long flags;
3046
3047 port_gid_table = &iboe->gids[port_num - 1];
3048 spin_lock_irqsave(&iboe->lock, flags);
3049 memcpy(gid, &port_gid_table->gids[index].gid, sizeof(*gid));
3050 *gid_type = port_gid_table->gids[index].gid_type;
3051 spin_unlock_irqrestore(&iboe->lock, flags);
3052 if (!memcmp(gid, &zgid, sizeof(*gid)))
3053 return -ENOENT;
3054
3055 return 0;
3056}
3057
3058#define MLX4_ROCEV2_QP1_SPORT 0xC000
3059static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_ud_wr *wr,
3060 void *wqe, unsigned *mlx_seg_len)
3061{
3062 struct ib_device *ib_dev = sqp->qp.ibqp.device;
3063 struct mlx4_ib_dev *ibdev = to_mdev(ib_dev);
3064 struct mlx4_wqe_mlx_seg *mlx = wqe;
3065 struct mlx4_wqe_ctrl_seg *ctrl = wqe;
3066 struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
3067 struct mlx4_ib_ah *ah = to_mah(wr->ah);
3068 union ib_gid sgid;
3069 u16 pkey;
3070 int send_size;
3071 int header_size;
3072 int spc;
3073 int i;
3074 int err = 0;
3075 u16 vlan = 0xffff;
3076 bool is_eth;
3077 bool is_vlan = false;
3078 bool is_grh;
3079 bool is_udp = false;
3080 int ip_version = 0;
3081
3082 send_size = 0;
3083 for (i = 0; i < wr->wr.num_sge; ++i)
3084 send_size += wr->wr.sg_list[i].length;
3085
3086 is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
3087 is_grh = mlx4_ib_ah_grh_present(ah);
3088 if (is_eth) {
3089 enum ib_gid_type gid_type;
3090 if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3091
3092
3093
3094 err = mlx4_get_roce_gid_from_slave(to_mdev(ib_dev)->dev,
3095 be32_to_cpu(ah->av.ib.port_pd) >> 24,
3096 ah->av.ib.gid_index, &sgid.raw[0]);
3097 if (err)
3098 return err;
3099 } else {
3100 err = fill_gid_by_hw_index(ibdev, sqp->qp.port,
3101 ah->av.ib.gid_index,
3102 &sgid, &gid_type);
3103 if (!err) {
3104 is_udp = gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
3105 if (is_udp) {
3106 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid))
3107 ip_version = 4;
3108 else
3109 ip_version = 6;
3110 is_grh = false;
3111 }
3112 } else {
3113 return err;
3114 }
3115 }
3116 if (ah->av.eth.vlan != cpu_to_be16(0xffff)) {
3117 vlan = be16_to_cpu(ah->av.eth.vlan) & 0x0fff;
3118 is_vlan = 1;
3119 }
3120 }
3121 err = ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh,
3122 ip_version, is_udp, 0, &sqp->ud_header);
3123 if (err)
3124 return err;
3125
3126 if (!is_eth) {
3127 sqp->ud_header.lrh.service_level =
3128 be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
3129 sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
3130 sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
3131 }
3132
3133 if (is_grh || (ip_version == 6)) {
3134 sqp->ud_header.grh.traffic_class =
3135 (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3136 sqp->ud_header.grh.flow_label =
3137 ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
3138 sqp->ud_header.grh.hop_limit = ah->av.ib.hop_limit;
3139 if (is_eth) {
3140 memcpy(sqp->ud_header.grh.source_gid.raw, sgid.raw, 16);
3141 } else {
3142 if (mlx4_is_mfunc(to_mdev(ib_dev)->dev)) {
3143
3144
3145
3146
3147 sqp->ud_header.grh.source_gid.global.subnet_prefix =
3148 cpu_to_be64(atomic64_read(&(to_mdev(ib_dev)->sriov.
3149 demux[sqp->qp.port - 1].
3150 subnet_prefix)));
3151 sqp->ud_header.grh.source_gid.global.interface_id =
3152 to_mdev(ib_dev)->sriov.demux[sqp->qp.port - 1].
3153 guid_cache[ah->av.ib.gid_index];
3154 } else {
3155 ib_get_cached_gid(ib_dev,
3156 be32_to_cpu(ah->av.ib.port_pd) >> 24,
3157 ah->av.ib.gid_index,
3158 &sqp->ud_header.grh.source_gid, NULL);
3159 }
3160 }
3161 memcpy(sqp->ud_header.grh.destination_gid.raw,
3162 ah->av.ib.dgid, 16);
3163 }
3164
3165 if (ip_version == 4) {
3166 sqp->ud_header.ip4.tos =
3167 (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
3168 sqp->ud_header.ip4.id = 0;
3169 sqp->ud_header.ip4.frag_off = htons(IP_DF);
3170 sqp->ud_header.ip4.ttl = ah->av.eth.hop_limit;
3171
3172 memcpy(&sqp->ud_header.ip4.saddr,
3173 sgid.raw + 12, 4);
3174 memcpy(&sqp->ud_header.ip4.daddr, ah->av.ib.dgid + 12, 4);
3175 sqp->ud_header.ip4.check = ib_ud_ip4_csum(&sqp->ud_header);
3176 }
3177
3178 if (is_udp) {
3179 sqp->ud_header.udp.dport = htons(ROCE_V2_UDP_DPORT);
3180 sqp->ud_header.udp.sport = htons(MLX4_ROCEV2_QP1_SPORT);
3181 sqp->ud_header.udp.csum = 0;
3182 }
3183
3184 mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
3185
3186 if (!is_eth) {
3187 mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
3188 (sqp->ud_header.lrh.destination_lid ==
3189 IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
3190 (sqp->ud_header.lrh.service_level << 8));
3191 if (ah->av.ib.port_pd & cpu_to_be32(0x80000000))
3192 mlx->flags |= cpu_to_be32(0x1);
3193 mlx->rlid = sqp->ud_header.lrh.destination_lid;
3194 }
3195
3196 switch (wr->wr.opcode) {
3197 case IB_WR_SEND:
3198 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
3199 sqp->ud_header.immediate_present = 0;
3200 break;
3201 case IB_WR_SEND_WITH_IMM:
3202 sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
3203 sqp->ud_header.immediate_present = 1;
3204 sqp->ud_header.immediate_data = wr->wr.ex.imm_data;
3205 break;
3206 default:
3207 return -EINVAL;
3208 }
3209
3210 if (is_eth) {
3211 struct in6_addr in6;
3212 u16 ether_type;
3213 u16 pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 29) << 13;
3214
3215 ether_type = (!is_udp) ? ETH_P_IBOE:
3216 (ip_version == 4 ? ETH_P_IP : ETH_P_IPV6);
3217
3218 mlx->sched_prio = cpu_to_be16(pcp);
3219
3220 ether_addr_copy(sqp->ud_header.eth.smac_h, ah->av.eth.s_mac);
3221 memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
3222 memcpy(&ctrl->srcrb_flags16[0], ah->av.eth.mac, 2);
3223 memcpy(&ctrl->imm, ah->av.eth.mac + 2, 4);
3224 memcpy(&in6, sgid.raw, sizeof(in6));
3225
3226
3227 if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
3228 mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
3229 if (!is_vlan) {
3230 sqp->ud_header.eth.type = cpu_to_be16(ether_type);
3231 } else {
3232 sqp->ud_header.vlan.type = cpu_to_be16(ether_type);
3233 sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
3234 }
3235 } else {
3236 sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 :
3237 sl_to_vl(to_mdev(ib_dev),
3238 sqp->ud_header.lrh.service_level,
3239 sqp->qp.port);
3240 if (sqp->qp.ibqp.qp_num && sqp->ud_header.lrh.virtual_lane == 15)
3241 return -EINVAL;
3242 if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
3243 sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
3244 }
3245 sqp->ud_header.bth.solicited_event = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
3246 if (!sqp->qp.ibqp.qp_num)
3247 ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
3248 else
3249 ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->pkey_index, &pkey);
3250 sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
3251 sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
3252 sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
3253 sqp->ud_header.deth.qkey = cpu_to_be32(wr->remote_qkey & 0x80000000 ?
3254 sqp->qkey : wr->remote_qkey);
3255 sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
3256
3257 header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
3258
3259 if (0) {
3260 pr_err("built UD header of size %d:\n", header_size);
3261 for (i = 0; i < header_size / 4; ++i) {
3262 if (i % 8 == 0)
3263 pr_err(" [%02x] ", i * 4);
3264 pr_cont(" %08x",
3265 be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
3266 if ((i + 1) % 8 == 0)
3267 pr_cont("\n");
3268 }
3269 pr_err("\n");
3270 }
3271
3272
3273
3274
3275
3276
3277
3278 spc = MLX4_INLINE_ALIGN -
3279 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3280 if (header_size <= spc) {
3281 inl->byte_count = cpu_to_be32(1 << 31 | header_size);
3282 memcpy(inl + 1, sqp->header_buf, header_size);
3283 i = 1;
3284 } else {
3285 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3286 memcpy(inl + 1, sqp->header_buf, spc);
3287
3288 inl = (void *) (inl + 1) + spc;
3289 memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303 wmb();
3304 inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
3305 i = 2;
3306 }
3307
3308 *mlx_seg_len =
3309 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
3310 return 0;
3311}
3312
3313static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
3314{
3315 unsigned cur;
3316 struct mlx4_ib_cq *cq;
3317
3318 cur = wq->head - wq->tail;
3319 if (likely(cur + nreq < wq->max_post))
3320 return 0;
3321
3322 cq = to_mcq(ib_cq);
3323 spin_lock(&cq->lock);
3324 cur = wq->head - wq->tail;
3325 spin_unlock(&cq->lock);
3326
3327 return cur + nreq >= wq->max_post;
3328}
3329
3330static __be32 convert_access(int acc)
3331{
3332 return (acc & IB_ACCESS_REMOTE_ATOMIC ?
3333 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_ATOMIC) : 0) |
3334 (acc & IB_ACCESS_REMOTE_WRITE ?
3335 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_WRITE) : 0) |
3336 (acc & IB_ACCESS_REMOTE_READ ?
3337 cpu_to_be32(MLX4_WQE_FMR_AND_BIND_PERM_REMOTE_READ) : 0) |
3338 (acc & IB_ACCESS_LOCAL_WRITE ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE) : 0) |
3339 cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
3340}
3341
3342static void set_reg_seg(struct mlx4_wqe_fmr_seg *fseg,
3343 struct ib_reg_wr *wr)
3344{
3345 struct mlx4_ib_mr *mr = to_mmr(wr->mr);
3346
3347 fseg->flags = convert_access(wr->access);
3348 fseg->mem_key = cpu_to_be32(wr->key);
3349 fseg->buf_list = cpu_to_be64(mr->page_map);
3350 fseg->start_addr = cpu_to_be64(mr->ibmr.iova);
3351 fseg->reg_len = cpu_to_be64(mr->ibmr.length);
3352 fseg->offset = 0;
3353 fseg->page_size = cpu_to_be32(ilog2(mr->ibmr.page_size));
3354 fseg->reserved[0] = 0;
3355 fseg->reserved[1] = 0;
3356}
3357
3358static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
3359{
3360 memset(iseg, 0, sizeof(*iseg));
3361 iseg->mem_key = cpu_to_be32(rkey);
3362}
3363
3364static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
3365 u64 remote_addr, u32 rkey)
3366{
3367 rseg->raddr = cpu_to_be64(remote_addr);
3368 rseg->rkey = cpu_to_be32(rkey);
3369 rseg->reserved = 0;
3370}
3371
3372static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg,
3373 struct ib_atomic_wr *wr)
3374{
3375 if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
3376 aseg->swap_add = cpu_to_be64(wr->swap);
3377 aseg->compare = cpu_to_be64(wr->compare_add);
3378 } else if (wr->wr.opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
3379 aseg->swap_add = cpu_to_be64(wr->compare_add);
3380 aseg->compare = cpu_to_be64(wr->compare_add_mask);
3381 } else {
3382 aseg->swap_add = cpu_to_be64(wr->compare_add);
3383 aseg->compare = 0;
3384 }
3385
3386}
3387
3388static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
3389 struct ib_atomic_wr *wr)
3390{
3391 aseg->swap_add = cpu_to_be64(wr->swap);
3392 aseg->swap_add_mask = cpu_to_be64(wr->swap_mask);
3393 aseg->compare = cpu_to_be64(wr->compare_add);
3394 aseg->compare_mask = cpu_to_be64(wr->compare_add_mask);
3395}
3396
3397static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
3398 struct ib_ud_wr *wr)
3399{
3400 memcpy(dseg->av, &to_mah(wr->ah)->av, sizeof (struct mlx4_av));
3401 dseg->dqpn = cpu_to_be32(wr->remote_qpn);
3402 dseg->qkey = cpu_to_be32(wr->remote_qkey);
3403 dseg->vlan = to_mah(wr->ah)->av.eth.vlan;
3404 memcpy(dseg->mac, to_mah(wr->ah)->av.eth.mac, 6);
3405}
3406
3407static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev,
3408 struct mlx4_wqe_datagram_seg *dseg,
3409 struct ib_ud_wr *wr,
3410 enum mlx4_ib_qp_type qpt)
3411{
3412 union mlx4_ext_av *av = &to_mah(wr->ah)->av;
3413 struct mlx4_av sqp_av = {0};
3414 int port = *((u8 *) &av->ib.port_pd) & 0x3;
3415
3416
3417 sqp_av.port_pd = av->ib.port_pd | cpu_to_be32(0x80000000);
3418 sqp_av.g_slid = av->ib.g_slid & 0x7f;
3419 sqp_av.sl_tclass_flowlabel = av->ib.sl_tclass_flowlabel &
3420 cpu_to_be32(0xf0000000);
3421
3422 memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av));
3423 if (qpt == MLX4_IB_QPT_PROXY_GSI)
3424 dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp1_tunnel);
3425 else
3426 dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp0_tunnel);
3427
3428 dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY);
3429}
3430
3431static void build_tunnel_header(struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len)
3432{
3433 struct mlx4_wqe_inline_seg *inl = wqe;
3434 struct mlx4_ib_tunnel_header hdr;
3435 struct mlx4_ib_ah *ah = to_mah(wr->ah);
3436 int spc;
3437 int i;
3438
3439 memcpy(&hdr.av, &ah->av, sizeof hdr.av);
3440 hdr.remote_qpn = cpu_to_be32(wr->remote_qpn);
3441 hdr.pkey_index = cpu_to_be16(wr->pkey_index);
3442 hdr.qkey = cpu_to_be32(wr->remote_qkey);
3443 memcpy(hdr.mac, ah->av.eth.mac, 6);
3444 hdr.vlan = ah->av.eth.vlan;
3445
3446 spc = MLX4_INLINE_ALIGN -
3447 ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
3448 if (sizeof (hdr) <= spc) {
3449 memcpy(inl + 1, &hdr, sizeof (hdr));
3450 wmb();
3451 inl->byte_count = cpu_to_be32(1 << 31 | sizeof (hdr));
3452 i = 1;
3453 } else {
3454 memcpy(inl + 1, &hdr, spc);
3455 wmb();
3456 inl->byte_count = cpu_to_be32(1 << 31 | spc);
3457
3458 inl = (void *) (inl + 1) + spc;
3459 memcpy(inl + 1, (void *) &hdr + spc, sizeof (hdr) - spc);
3460 wmb();
3461 inl->byte_count = cpu_to_be32(1 << 31 | (sizeof (hdr) - spc));
3462 i = 2;
3463 }
3464
3465 *mlx_seg_len =
3466 ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + sizeof (hdr), 16);
3467}
3468
3469static void set_mlx_icrc_seg(void *dseg)
3470{
3471 u32 *t = dseg;
3472 struct mlx4_wqe_inline_seg *iseg = dseg;
3473
3474 t[1] = 0;
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484 wmb();
3485
3486 iseg->byte_count = cpu_to_be32((1 << 31) | 4);
3487}
3488
3489static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3490{
3491 dseg->lkey = cpu_to_be32(sg->lkey);
3492 dseg->addr = cpu_to_be64(sg->addr);
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502 wmb();
3503
3504 dseg->byte_count = cpu_to_be32(sg->length);
3505}
3506
3507static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
3508{
3509 dseg->byte_count = cpu_to_be32(sg->length);
3510 dseg->lkey = cpu_to_be32(sg->lkey);
3511 dseg->addr = cpu_to_be64(sg->addr);
3512}
3513
3514static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_ud_wr *wr,
3515 struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
3516 __be32 *lso_hdr_sz, __be32 *blh)
3517{
3518 unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16);
3519
3520 if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
3521 *blh = cpu_to_be32(1 << 6);
3522
3523 if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
3524 wr->wr.num_sge > qp->sq.max_gs - (halign >> 4)))
3525 return -EINVAL;
3526
3527 memcpy(wqe->header, wr->header, wr->hlen);
3528
3529 *lso_hdr_sz = cpu_to_be32(wr->mss << 16 | wr->hlen);
3530 *lso_seg_len = halign;
3531 return 0;
3532}
3533
3534static __be32 send_ieth(struct ib_send_wr *wr)
3535{
3536 switch (wr->opcode) {
3537 case IB_WR_SEND_WITH_IMM:
3538 case IB_WR_RDMA_WRITE_WITH_IMM:
3539 return wr->ex.imm_data;
3540
3541 case IB_WR_SEND_WITH_INV:
3542 return cpu_to_be32(wr->ex.invalidate_rkey);
3543
3544 default:
3545 return 0;
3546 }
3547}
3548
3549static void add_zero_len_inline(void *wqe)
3550{
3551 struct mlx4_wqe_inline_seg *inl = wqe;
3552 memset(wqe, 0, 16);
3553 inl->byte_count = cpu_to_be32(1 << 31);
3554}
3555
3556int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3557 struct ib_send_wr **bad_wr)
3558{
3559 struct mlx4_ib_qp *qp = to_mqp(ibqp);
3560 void *wqe;
3561 struct mlx4_wqe_ctrl_seg *ctrl;
3562 struct mlx4_wqe_data_seg *dseg;
3563 unsigned long flags;
3564 int nreq;
3565 int err = 0;
3566 unsigned ind;
3567 int uninitialized_var(stamp);
3568 int uninitialized_var(size);
3569 unsigned uninitialized_var(seglen);
3570 __be32 dummy;
3571 __be32 *lso_wqe;
3572 __be32 uninitialized_var(lso_hdr_sz);
3573 __be32 blh;
3574 int i;
3575 struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3576
3577 if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI) {
3578 struct mlx4_ib_sqp *sqp = to_msqp(qp);
3579
3580 if (sqp->roce_v2_gsi) {
3581 struct mlx4_ib_ah *ah = to_mah(ud_wr(wr)->ah);
3582 enum ib_gid_type gid_type;
3583 union ib_gid gid;
3584
3585 if (!fill_gid_by_hw_index(mdev, sqp->qp.port,
3586 ah->av.ib.gid_index,
3587 &gid, &gid_type))
3588 qp = (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
3589 to_mqp(sqp->roce_v2_gsi) : qp;
3590 else
3591 pr_err("Failed to get gid at index %d. RoCEv2 will not work properly\n",
3592 ah->av.ib.gid_index);
3593 }
3594 }
3595
3596 spin_lock_irqsave(&qp->sq.lock, flags);
3597 if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
3598 err = -EIO;
3599 *bad_wr = wr;
3600 nreq = 0;
3601 goto out;
3602 }
3603
3604 ind = qp->sq_next_wqe;
3605
3606 for (nreq = 0; wr; ++nreq, wr = wr->next) {
3607 lso_wqe = &dummy;
3608 blh = 0;
3609
3610 if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
3611 err = -ENOMEM;
3612 *bad_wr = wr;
3613 goto out;
3614 }
3615
3616 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
3617 err = -EINVAL;
3618 *bad_wr = wr;
3619 goto out;
3620 }
3621
3622 ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
3623 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
3624
3625 ctrl->srcrb_flags =
3626 (wr->send_flags & IB_SEND_SIGNALED ?
3627 cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
3628 (wr->send_flags & IB_SEND_SOLICITED ?
3629 cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
3630 ((wr->send_flags & IB_SEND_IP_CSUM) ?
3631 cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
3632 MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
3633 qp->sq_signal_bits;
3634
3635 ctrl->imm = send_ieth(wr);
3636
3637 wqe += sizeof *ctrl;
3638 size = sizeof *ctrl / 16;
3639
3640 switch (qp->mlx4_ib_qp_type) {
3641 case MLX4_IB_QPT_RC:
3642 case MLX4_IB_QPT_UC:
3643 switch (wr->opcode) {
3644 case IB_WR_ATOMIC_CMP_AND_SWP:
3645 case IB_WR_ATOMIC_FETCH_AND_ADD:
3646 case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
3647 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3648 atomic_wr(wr)->rkey);
3649 wqe += sizeof (struct mlx4_wqe_raddr_seg);
3650
3651 set_atomic_seg(wqe, atomic_wr(wr));
3652 wqe += sizeof (struct mlx4_wqe_atomic_seg);
3653
3654 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3655 sizeof (struct mlx4_wqe_atomic_seg)) / 16;
3656
3657 break;
3658
3659 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
3660 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
3661 atomic_wr(wr)->rkey);
3662 wqe += sizeof (struct mlx4_wqe_raddr_seg);
3663
3664 set_masked_atomic_seg(wqe, atomic_wr(wr));
3665 wqe += sizeof (struct mlx4_wqe_masked_atomic_seg);
3666
3667 size += (sizeof (struct mlx4_wqe_raddr_seg) +
3668 sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
3669
3670 break;
3671
3672 case IB_WR_RDMA_READ:
3673 case IB_WR_RDMA_WRITE:
3674 case IB_WR_RDMA_WRITE_WITH_IMM:
3675 set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,
3676 rdma_wr(wr)->rkey);
3677 wqe += sizeof (struct mlx4_wqe_raddr_seg);
3678 size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
3679 break;
3680
3681 case IB_WR_LOCAL_INV:
3682 ctrl->srcrb_flags |=
3683 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3684 set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
3685 wqe += sizeof (struct mlx4_wqe_local_inval_seg);
3686 size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
3687 break;
3688
3689 case IB_WR_REG_MR:
3690 ctrl->srcrb_flags |=
3691 cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
3692 set_reg_seg(wqe, reg_wr(wr));
3693 wqe += sizeof(struct mlx4_wqe_fmr_seg);
3694 size += sizeof(struct mlx4_wqe_fmr_seg) / 16;
3695 break;
3696
3697 default:
3698
3699 break;
3700 }
3701 break;
3702
3703 case MLX4_IB_QPT_TUN_SMI_OWNER:
3704 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3705 ctrl, &seglen);
3706 if (unlikely(err)) {
3707 *bad_wr = wr;
3708 goto out;
3709 }
3710 wqe += seglen;
3711 size += seglen / 16;
3712 break;
3713 case MLX4_IB_QPT_TUN_SMI:
3714 case MLX4_IB_QPT_TUN_GSI:
3715
3716 set_datagram_seg(wqe, ud_wr(wr));
3717
3718 *(__be32 *) wqe |= cpu_to_be32(0x80000000);
3719 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3720 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3721 break;
3722 case MLX4_IB_QPT_UD:
3723 set_datagram_seg(wqe, ud_wr(wr));
3724 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3725 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3726
3727 if (wr->opcode == IB_WR_LSO) {
3728 err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen,
3729 &lso_hdr_sz, &blh);
3730 if (unlikely(err)) {
3731 *bad_wr = wr;
3732 goto out;
3733 }
3734 lso_wqe = (__be32 *) wqe;
3735 wqe += seglen;
3736 size += seglen / 16;
3737 }
3738 break;
3739
3740 case MLX4_IB_QPT_PROXY_SMI_OWNER:
3741 err = build_sriov_qp0_header(to_msqp(qp), ud_wr(wr),
3742 ctrl, &seglen);
3743 if (unlikely(err)) {
3744 *bad_wr = wr;
3745 goto out;
3746 }
3747 wqe += seglen;
3748 size += seglen / 16;
3749
3750 add_zero_len_inline(wqe);
3751 wqe += 16;
3752 size++;
3753 build_tunnel_header(ud_wr(wr), wqe, &seglen);
3754 wqe += seglen;
3755 size += seglen / 16;
3756 break;
3757 case MLX4_IB_QPT_PROXY_SMI:
3758 case MLX4_IB_QPT_PROXY_GSI:
3759
3760
3761
3762
3763 set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe,
3764 ud_wr(wr),
3765 qp->mlx4_ib_qp_type);
3766 wqe += sizeof (struct mlx4_wqe_datagram_seg);
3767 size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
3768 build_tunnel_header(ud_wr(wr), wqe, &seglen);
3769 wqe += seglen;
3770 size += seglen / 16;
3771 break;
3772
3773 case MLX4_IB_QPT_SMI:
3774 case MLX4_IB_QPT_GSI:
3775 err = build_mlx_header(to_msqp(qp), ud_wr(wr), ctrl,
3776 &seglen);
3777 if (unlikely(err)) {
3778 *bad_wr = wr;
3779 goto out;
3780 }
3781 wqe += seglen;
3782 size += seglen / 16;
3783 break;
3784
3785 default:
3786 break;
3787 }
3788
3789
3790
3791
3792
3793
3794
3795
3796 dseg = wqe;
3797 dseg += wr->num_sge - 1;
3798 size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
3799
3800
3801 if (unlikely(qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
3802 qp->mlx4_ib_qp_type == MLX4_IB_QPT_GSI ||
3803 qp->mlx4_ib_qp_type &
3804 (MLX4_IB_QPT_PROXY_SMI_OWNER | MLX4_IB_QPT_TUN_SMI_OWNER))) {
3805 set_mlx_icrc_seg(dseg + 1);
3806 size += sizeof (struct mlx4_wqe_data_seg) / 16;
3807 }
3808
3809 for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
3810 set_data_seg(dseg, wr->sg_list + i);
3811
3812
3813
3814
3815
3816
3817 wmb();
3818 *lso_wqe = lso_hdr_sz;
3819
3820 ctrl->qpn_vlan.fence_size = (wr->send_flags & IB_SEND_FENCE ?
3821 MLX4_WQE_CTRL_FENCE : 0) | size;
3822
3823
3824
3825
3826
3827
3828 wmb();
3829
3830 if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
3831 *bad_wr = wr;
3832 err = -EINVAL;
3833 goto out;
3834 }
3835
3836 ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
3837 (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
3838
3839 stamp = ind + qp->sq_spare_wqes;
3840 ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851 if (wr->next) {
3852 stamp_send_wqe(qp, stamp, size * 16);
3853 ind = pad_wraparound(qp, ind);
3854 }
3855 }
3856
3857out:
3858 if (likely(nreq)) {
3859 qp->sq.head += nreq;
3860
3861
3862
3863
3864
3865 wmb();
3866
3867 writel(qp->doorbell_qpn,
3868 to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
3869
3870
3871
3872
3873
3874 mmiowb();
3875
3876 stamp_send_wqe(qp, stamp, size * 16);
3877
3878 ind = pad_wraparound(qp, ind);
3879 qp->sq_next_wqe = ind;
3880 }
3881
3882 spin_unlock_irqrestore(&qp->sq.lock, flags);
3883
3884 return err;
3885}
3886
3887int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3888 struct ib_recv_wr **bad_wr)
3889{
3890 struct mlx4_ib_qp *qp = to_mqp(ibqp);
3891 struct mlx4_wqe_data_seg *scat;
3892 unsigned long flags;
3893 int err = 0;
3894 int nreq;
3895 int ind;
3896 int max_gs;
3897 int i;
3898 struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
3899
3900 max_gs = qp->rq.max_gs;
3901 spin_lock_irqsave(&qp->rq.lock, flags);
3902
3903 if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
3904 err = -EIO;
3905 *bad_wr = wr;
3906 nreq = 0;
3907 goto out;
3908 }
3909
3910 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
3911
3912 for (nreq = 0; wr; ++nreq, wr = wr->next) {
3913 if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
3914 err = -ENOMEM;
3915 *bad_wr = wr;
3916 goto out;
3917 }
3918
3919 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
3920 err = -EINVAL;
3921 *bad_wr = wr;
3922 goto out;
3923 }
3924
3925 scat = get_recv_wqe(qp, ind);
3926
3927 if (qp->mlx4_ib_qp_type & (MLX4_IB_QPT_PROXY_SMI_OWNER |
3928 MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
3929 ib_dma_sync_single_for_device(ibqp->device,
3930 qp->sqp_proxy_rcv[ind].map,
3931 sizeof (struct mlx4_ib_proxy_sqp_hdr),
3932 DMA_FROM_DEVICE);
3933 scat->byte_count =
3934 cpu_to_be32(sizeof (struct mlx4_ib_proxy_sqp_hdr));
3935
3936 scat->lkey = cpu_to_be32(wr->sg_list->lkey);
3937 scat->addr = cpu_to_be64(qp->sqp_proxy_rcv[ind].map);
3938 scat++;
3939 max_gs--;
3940 }
3941
3942 for (i = 0; i < wr->num_sge; ++i)
3943 __set_data_seg(scat + i, wr->sg_list + i);
3944
3945 if (i < max_gs) {
3946 scat[i].byte_count = 0;
3947 scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
3948 scat[i].addr = 0;
3949 }
3950
3951 qp->rq.wrid[ind] = wr->wr_id;
3952
3953 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
3954 }
3955
3956out:
3957 if (likely(nreq)) {
3958 qp->rq.head += nreq;
3959
3960
3961
3962
3963
3964 wmb();
3965
3966 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
3967 }
3968
3969 spin_unlock_irqrestore(&qp->rq.lock, flags);
3970
3971 return err;
3972}
3973
3974static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
3975{
3976 switch (mlx4_state) {
3977 case MLX4_QP_STATE_RST: return IB_QPS_RESET;
3978 case MLX4_QP_STATE_INIT: return IB_QPS_INIT;
3979 case MLX4_QP_STATE_RTR: return IB_QPS_RTR;
3980 case MLX4_QP_STATE_RTS: return IB_QPS_RTS;
3981 case MLX4_QP_STATE_SQ_DRAINING:
3982 case MLX4_QP_STATE_SQD: return IB_QPS_SQD;
3983 case MLX4_QP_STATE_SQER: return IB_QPS_SQE;
3984 case MLX4_QP_STATE_ERR: return IB_QPS_ERR;
3985 default: return -1;
3986 }
3987}
3988
3989static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
3990{
3991 switch (mlx4_mig_state) {
3992 case MLX4_QP_PM_ARMED: return IB_MIG_ARMED;
3993 case MLX4_QP_PM_REARM: return IB_MIG_REARM;
3994 case MLX4_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
3995 default: return -1;
3996 }
3997}
3998
3999static int to_ib_qp_access_flags(int mlx4_flags)
4000{
4001 int ib_flags = 0;
4002
4003 if (mlx4_flags & MLX4_QP_BIT_RRE)
4004 ib_flags |= IB_ACCESS_REMOTE_READ;
4005 if (mlx4_flags & MLX4_QP_BIT_RWE)
4006 ib_flags |= IB_ACCESS_REMOTE_WRITE;
4007 if (mlx4_flags & MLX4_QP_BIT_RAE)
4008 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
4009
4010 return ib_flags;
4011}
4012
4013static void to_rdma_ah_attr(struct mlx4_ib_dev *ibdev,
4014 struct rdma_ah_attr *ah_attr,
4015 struct mlx4_qp_path *path)
4016{
4017 struct mlx4_dev *dev = ibdev->dev;
4018 u8 port_num = path->sched_queue & 0x40 ? 2 : 1;
4019
4020 memset(ah_attr, 0, sizeof(*ah_attr));
4021 ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port_num);
4022 if (port_num == 0 || port_num > dev->caps.num_ports)
4023 return;
4024
4025 if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE)
4026 rdma_ah_set_sl(ah_attr, ((path->sched_queue >> 3) & 0x7) |
4027 ((path->sched_queue & 4) << 1));
4028 else
4029 rdma_ah_set_sl(ah_attr, (path->sched_queue >> 2) & 0xf);
4030 rdma_ah_set_port_num(ah_attr, port_num);
4031
4032 rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
4033 rdma_ah_set_path_bits(ah_attr, path->grh_mylmc & 0x7f);
4034 rdma_ah_set_static_rate(ah_attr,
4035 path->static_rate ? path->static_rate - 5 : 0);
4036 if (path->grh_mylmc & (1 << 7)) {
4037 rdma_ah_set_grh(ah_attr, NULL,
4038 be32_to_cpu(path->tclass_flowlabel) & 0xfffff,
4039 path->mgid_index,
4040 path->hop_limit,
4041 (be32_to_cpu(path->tclass_flowlabel)
4042 >> 20) & 0xff);
4043 rdma_ah_set_dgid_raw(ah_attr, path->rgid);
4044 }
4045}
4046
4047int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
4048 struct ib_qp_init_attr *qp_init_attr)
4049{
4050 struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
4051 struct mlx4_ib_qp *qp = to_mqp(ibqp);
4052 struct mlx4_qp_context context;
4053 int mlx4_state;
4054 int err = 0;
4055
4056 if (ibqp->rwq_ind_tbl)
4057 return -EOPNOTSUPP;
4058
4059 mutex_lock(&qp->mutex);
4060
4061 if (qp->state == IB_QPS_RESET) {
4062 qp_attr->qp_state = IB_QPS_RESET;
4063 goto done;
4064 }
4065
4066 err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
4067 if (err) {
4068 err = -EINVAL;
4069 goto out;
4070 }
4071
4072 mlx4_state = be32_to_cpu(context.flags) >> 28;
4073
4074 qp->state = to_ib_qp_state(mlx4_state);
4075 qp_attr->qp_state = qp->state;
4076 qp_attr->path_mtu = context.mtu_msgmax >> 5;
4077 qp_attr->path_mig_state =
4078 to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
4079 qp_attr->qkey = be32_to_cpu(context.qkey);
4080 qp_attr->rq_psn = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
4081 qp_attr->sq_psn = be32_to_cpu(context.next_send_psn) & 0xffffff;
4082 qp_attr->dest_qp_num = be32_to_cpu(context.remote_qpn) & 0xffffff;
4083 qp_attr->qp_access_flags =
4084 to_ib_qp_access_flags(be32_to_cpu(context.params2));
4085
4086 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
4087 to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
4088 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
4089 qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
4090 qp_attr->alt_port_num =
4091 rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
4092 }
4093
4094 qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
4095 if (qp_attr->qp_state == IB_QPS_INIT)
4096 qp_attr->port_num = qp->port;
4097 else
4098 qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
4099
4100
4101 qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
4102
4103 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
4104
4105 qp_attr->max_dest_rd_atomic =
4106 1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
4107 qp_attr->min_rnr_timer =
4108 (be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
4109 qp_attr->timeout = context.pri_path.ackto >> 3;
4110 qp_attr->retry_cnt = (be32_to_cpu(context.params1) >> 16) & 0x7;
4111 qp_attr->rnr_retry = (be32_to_cpu(context.params1) >> 13) & 0x7;
4112 qp_attr->alt_timeout = context.alt_path.ackto >> 3;
4113
4114done:
4115 qp_attr->cur_qp_state = qp_attr->qp_state;
4116 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
4117 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
4118
4119 if (!ibqp->uobject) {
4120 qp_attr->cap.max_send_wr = qp->sq.wqe_cnt;
4121 qp_attr->cap.max_send_sge = qp->sq.max_gs;
4122 } else {
4123 qp_attr->cap.max_send_wr = 0;
4124 qp_attr->cap.max_send_sge = 0;
4125 }
4126
4127
4128
4129
4130
4131 qp_attr->cap.max_inline_data = 0;
4132
4133 qp_init_attr->cap = qp_attr->cap;
4134
4135 qp_init_attr->create_flags = 0;
4136 if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
4137 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
4138
4139 if (qp->flags & MLX4_IB_QP_LSO)
4140 qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
4141
4142 if (qp->flags & MLX4_IB_QP_NETIF)
4143 qp_init_attr->create_flags |= IB_QP_CREATE_NETIF_QP;
4144
4145 qp_init_attr->sq_sig_type =
4146 qp->sq_signal_bits == cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) ?
4147 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
4148
4149out:
4150 mutex_unlock(&qp->mutex);
4151 return err;
4152}
4153
4154struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
4155 struct ib_wq_init_attr *init_attr,
4156 struct ib_udata *udata)
4157{
4158 struct mlx4_ib_dev *dev;
4159 struct ib_qp_init_attr ib_qp_init_attr;
4160 struct mlx4_ib_qp *qp;
4161 struct mlx4_ib_create_wq ucmd;
4162 int err, required_cmd_sz;
4163
4164 if (!(udata && pd->uobject))
4165 return ERR_PTR(-EINVAL);
4166
4167 required_cmd_sz = offsetof(typeof(ucmd), comp_mask) +
4168 sizeof(ucmd.comp_mask);
4169 if (udata->inlen < required_cmd_sz) {
4170 pr_debug("invalid inlen\n");
4171 return ERR_PTR(-EINVAL);
4172 }
4173
4174 if (udata->inlen > sizeof(ucmd) &&
4175 !ib_is_udata_cleared(udata, sizeof(ucmd),
4176 udata->inlen - sizeof(ucmd))) {
4177 pr_debug("inlen is not supported\n");
4178 return ERR_PTR(-EOPNOTSUPP);
4179 }
4180
4181 if (udata->outlen)
4182 return ERR_PTR(-EOPNOTSUPP);
4183
4184 dev = to_mdev(pd->device);
4185
4186 if (init_attr->wq_type != IB_WQT_RQ) {
4187 pr_debug("unsupported wq type %d\n", init_attr->wq_type);
4188 return ERR_PTR(-EOPNOTSUPP);
4189 }
4190
4191 if (init_attr->create_flags) {
4192 pr_debug("unsupported create_flags %u\n",
4193 init_attr->create_flags);
4194 return ERR_PTR(-EOPNOTSUPP);
4195 }
4196
4197 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
4198 if (!qp)
4199 return ERR_PTR(-ENOMEM);
4200
4201 qp->pri.vid = 0xFFFF;
4202 qp->alt.vid = 0xFFFF;
4203
4204 memset(&ib_qp_init_attr, 0, sizeof(ib_qp_init_attr));
4205 ib_qp_init_attr.qp_context = init_attr->wq_context;
4206 ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;
4207 ib_qp_init_attr.cap.max_recv_wr = init_attr->max_wr;
4208 ib_qp_init_attr.cap.max_recv_sge = init_attr->max_sge;
4209 ib_qp_init_attr.recv_cq = init_attr->cq;
4210 ib_qp_init_attr.send_cq = ib_qp_init_attr.recv_cq;
4211
4212 err = create_qp_common(dev, pd, MLX4_IB_RWQ_SRC, &ib_qp_init_attr,
4213 udata, 0, &qp);
4214 if (err) {
4215 kfree(qp);
4216 return ERR_PTR(err);
4217 }
4218
4219 qp->ibwq.event_handler = init_attr->event_handler;
4220 qp->ibwq.wq_num = qp->mqp.qpn;
4221 qp->ibwq.state = IB_WQS_RESET;
4222
4223 return &qp->ibwq;
4224}
4225
4226static int ib_wq2qp_state(enum ib_wq_state state)
4227{
4228 switch (state) {
4229 case IB_WQS_RESET:
4230 return IB_QPS_RESET;
4231 case IB_WQS_RDY:
4232 return IB_QPS_RTR;
4233 default:
4234 return IB_QPS_ERR;
4235 }
4236}
4237
4238static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state)
4239{
4240 struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4241 enum ib_qp_state qp_cur_state;
4242 enum ib_qp_state qp_new_state;
4243 int attr_mask;
4244 int err;
4245
4246
4247
4248
4249 qp_cur_state = qp->state;
4250 qp_new_state = ib_wq2qp_state(new_state);
4251
4252 if (ib_wq2qp_state(new_state) == qp_cur_state)
4253 return 0;
4254
4255 if (new_state == IB_WQS_RDY) {
4256 struct ib_qp_attr attr = {};
4257
4258 attr.port_num = qp->port;
4259 attr_mask = IB_QP_PORT;
4260
4261 err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, &attr,
4262 attr_mask, IB_QPS_RESET, IB_QPS_INIT);
4263 if (err) {
4264 pr_debug("WQN=0x%06x failed to apply RST->INIT on the HW QP\n",
4265 ibwq->wq_num);
4266 return err;
4267 }
4268
4269 qp_cur_state = IB_QPS_INIT;
4270 }
4271
4272 attr_mask = 0;
4273 err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL, attr_mask,
4274 qp_cur_state, qp_new_state);
4275
4276 if (err && (qp_cur_state == IB_QPS_INIT)) {
4277 qp_new_state = IB_QPS_RESET;
4278 if (__mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL,
4279 attr_mask, IB_QPS_INIT, IB_QPS_RESET)) {
4280 pr_warn("WQN=0x%06x failed with reverting HW's resources failure\n",
4281 ibwq->wq_num);
4282 qp_new_state = IB_QPS_INIT;
4283 }
4284 }
4285
4286 qp->state = qp_new_state;
4287
4288 return err;
4289}
4290
4291int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,
4292 u32 wq_attr_mask, struct ib_udata *udata)
4293{
4294 struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4295 struct mlx4_ib_modify_wq ucmd = {};
4296 size_t required_cmd_sz;
4297 enum ib_wq_state cur_state, new_state;
4298 int err = 0;
4299
4300 required_cmd_sz = offsetof(typeof(ucmd), reserved) +
4301 sizeof(ucmd.reserved);
4302 if (udata->inlen < required_cmd_sz)
4303 return -EINVAL;
4304
4305 if (udata->inlen > sizeof(ucmd) &&
4306 !ib_is_udata_cleared(udata, sizeof(ucmd),
4307 udata->inlen - sizeof(ucmd)))
4308 return -EOPNOTSUPP;
4309
4310 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
4311 return -EFAULT;
4312
4313 if (ucmd.comp_mask || ucmd.reserved)
4314 return -EOPNOTSUPP;
4315
4316 if (wq_attr_mask & IB_WQ_FLAGS)
4317 return -EOPNOTSUPP;
4318
4319 cur_state = wq_attr_mask & IB_WQ_CUR_STATE ? wq_attr->curr_wq_state :
4320 ibwq->state;
4321 new_state = wq_attr_mask & IB_WQ_STATE ? wq_attr->wq_state : cur_state;
4322
4323 if (cur_state < IB_WQS_RESET || cur_state > IB_WQS_ERR ||
4324 new_state < IB_WQS_RESET || new_state > IB_WQS_ERR)
4325 return -EINVAL;
4326
4327 if ((new_state == IB_WQS_RDY) && (cur_state == IB_WQS_ERR))
4328 return -EINVAL;
4329
4330 if ((new_state == IB_WQS_ERR) && (cur_state == IB_WQS_RESET))
4331 return -EINVAL;
4332
4333
4334
4335
4336 mutex_lock(&qp->mutex);
4337
4338
4339
4340
4341 if (qp->rss_usecnt)
4342 err = _mlx4_ib_modify_wq(ibwq, new_state);
4343
4344 if (!err)
4345 ibwq->state = new_state;
4346
4347 mutex_unlock(&qp->mutex);
4348
4349 return err;
4350}
4351
4352int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
4353{
4354 struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
4355 struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
4356
4357 if (qp->counter_index)
4358 mlx4_ib_free_qp_counter(dev, qp);
4359
4360 destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, 1);
4361
4362 kfree(qp);
4363
4364 return 0;
4365}
4366
4367struct ib_rwq_ind_table
4368*mlx4_ib_create_rwq_ind_table(struct ib_device *device,
4369 struct ib_rwq_ind_table_init_attr *init_attr,
4370 struct ib_udata *udata)
4371{
4372 struct ib_rwq_ind_table *rwq_ind_table;
4373 struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};
4374 unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size;
4375 unsigned int base_wqn;
4376 size_t min_resp_len;
4377 int i;
4378 int err;
4379
4380 if (udata->inlen > 0 &&
4381 !ib_is_udata_cleared(udata, 0,
4382 udata->inlen))
4383 return ERR_PTR(-EOPNOTSUPP);
4384
4385 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
4386 if (udata->outlen && udata->outlen < min_resp_len)
4387 return ERR_PTR(-EINVAL);
4388
4389 if (ind_tbl_size >
4390 device->attrs.rss_caps.max_rwq_indirection_table_size) {
4391 pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",
4392 ind_tbl_size,
4393 device->attrs.rss_caps.max_rwq_indirection_table_size);
4394 return ERR_PTR(-EINVAL);
4395 }
4396
4397 base_wqn = init_attr->ind_tbl[0]->wq_num;
4398
4399 if (base_wqn % ind_tbl_size) {
4400 pr_debug("WQN=0x%x isn't aligned with indirection table size\n",
4401 base_wqn);
4402 return ERR_PTR(-EINVAL);
4403 }
4404
4405 for (i = 1; i < ind_tbl_size; i++) {
4406 if (++base_wqn != init_attr->ind_tbl[i]->wq_num) {
4407 pr_debug("indirection table's WQNs aren't consecutive\n");
4408 return ERR_PTR(-EINVAL);
4409 }
4410 }
4411
4412 rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL);
4413 if (!rwq_ind_table)
4414 return ERR_PTR(-ENOMEM);
4415
4416 if (udata->outlen) {
4417 resp.response_length = offsetof(typeof(resp), response_length) +
4418 sizeof(resp.response_length);
4419 err = ib_copy_to_udata(udata, &resp, resp.response_length);
4420 if (err)
4421 goto err;
4422 }
4423
4424 return rwq_ind_table;
4425
4426err:
4427 kfree(rwq_ind_table);
4428 return ERR_PTR(err);
4429}
4430
4431int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
4432{
4433 kfree(ib_rwq_ind_tbl);
4434 return 0;
4435}
4436