1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39#include <linux/errno.h>
40#include <linux/err.h>
41#include <linux/export.h>
42#include <linux/string.h>
43#include <linux/slab.h>
44#include <linux/in.h>
45#include <linux/in6.h>
46#include <net/addrconf.h>
47#include <linux/security.h>
48
49#include <rdma/ib_verbs.h>
50#include <rdma/ib_cache.h>
51#include <rdma/ib_addr.h>
52#include <rdma/rw.h>
53
54#include "core_priv.h"
55
56static int ib_resolve_eth_dmac(struct ib_device *device,
57 struct rdma_ah_attr *ah_attr);
58
59static const char * const ib_events[] = {
60 [IB_EVENT_CQ_ERR] = "CQ error",
61 [IB_EVENT_QP_FATAL] = "QP fatal error",
62 [IB_EVENT_QP_REQ_ERR] = "QP request error",
63 [IB_EVENT_QP_ACCESS_ERR] = "QP access error",
64 [IB_EVENT_COMM_EST] = "communication established",
65 [IB_EVENT_SQ_DRAINED] = "send queue drained",
66 [IB_EVENT_PATH_MIG] = "path migration successful",
67 [IB_EVENT_PATH_MIG_ERR] = "path migration error",
68 [IB_EVENT_DEVICE_FATAL] = "device fatal error",
69 [IB_EVENT_PORT_ACTIVE] = "port active",
70 [IB_EVENT_PORT_ERR] = "port error",
71 [IB_EVENT_LID_CHANGE] = "LID change",
72 [IB_EVENT_PKEY_CHANGE] = "P_key change",
73 [IB_EVENT_SM_CHANGE] = "SM change",
74 [IB_EVENT_SRQ_ERR] = "SRQ error",
75 [IB_EVENT_SRQ_LIMIT_REACHED] = "SRQ limit reached",
76 [IB_EVENT_QP_LAST_WQE_REACHED] = "last WQE reached",
77 [IB_EVENT_CLIENT_REREGISTER] = "client reregister",
78 [IB_EVENT_GID_CHANGE] = "GID changed",
79};
80
81const char *__attribute_const__ ib_event_msg(enum ib_event_type event)
82{
83 size_t index = event;
84
85 return (index < ARRAY_SIZE(ib_events) && ib_events[index]) ?
86 ib_events[index] : "unrecognized event";
87}
88EXPORT_SYMBOL(ib_event_msg);
89
90static const char * const wc_statuses[] = {
91 [IB_WC_SUCCESS] = "success",
92 [IB_WC_LOC_LEN_ERR] = "local length error",
93 [IB_WC_LOC_QP_OP_ERR] = "local QP operation error",
94 [IB_WC_LOC_EEC_OP_ERR] = "local EE context operation error",
95 [IB_WC_LOC_PROT_ERR] = "local protection error",
96 [IB_WC_WR_FLUSH_ERR] = "WR flushed",
97 [IB_WC_MW_BIND_ERR] = "memory management operation error",
98 [IB_WC_BAD_RESP_ERR] = "bad response error",
99 [IB_WC_LOC_ACCESS_ERR] = "local access error",
100 [IB_WC_REM_INV_REQ_ERR] = "invalid request error",
101 [IB_WC_REM_ACCESS_ERR] = "remote access error",
102 [IB_WC_REM_OP_ERR] = "remote operation error",
103 [IB_WC_RETRY_EXC_ERR] = "transport retry counter exceeded",
104 [IB_WC_RNR_RETRY_EXC_ERR] = "RNR retry counter exceeded",
105 [IB_WC_LOC_RDD_VIOL_ERR] = "local RDD violation error",
106 [IB_WC_REM_INV_RD_REQ_ERR] = "remote invalid RD request",
107 [IB_WC_REM_ABORT_ERR] = "operation aborted",
108 [IB_WC_INV_EECN_ERR] = "invalid EE context number",
109 [IB_WC_INV_EEC_STATE_ERR] = "invalid EE context state",
110 [IB_WC_FATAL_ERR] = "fatal error",
111 [IB_WC_RESP_TIMEOUT_ERR] = "response timeout error",
112 [IB_WC_GENERAL_ERR] = "general error",
113};
114
115const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status)
116{
117 size_t index = status;
118
119 return (index < ARRAY_SIZE(wc_statuses) && wc_statuses[index]) ?
120 wc_statuses[index] : "unrecognized status";
121}
122EXPORT_SYMBOL(ib_wc_status_msg);
123
124__attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
125{
126 switch (rate) {
127 case IB_RATE_2_5_GBPS: return 1;
128 case IB_RATE_5_GBPS: return 2;
129 case IB_RATE_10_GBPS: return 4;
130 case IB_RATE_20_GBPS: return 8;
131 case IB_RATE_30_GBPS: return 12;
132 case IB_RATE_40_GBPS: return 16;
133 case IB_RATE_60_GBPS: return 24;
134 case IB_RATE_80_GBPS: return 32;
135 case IB_RATE_120_GBPS: return 48;
136 case IB_RATE_14_GBPS: return 6;
137 case IB_RATE_56_GBPS: return 22;
138 case IB_RATE_112_GBPS: return 45;
139 case IB_RATE_168_GBPS: return 67;
140 case IB_RATE_25_GBPS: return 10;
141 case IB_RATE_100_GBPS: return 40;
142 case IB_RATE_200_GBPS: return 80;
143 case IB_RATE_300_GBPS: return 120;
144 case IB_RATE_28_GBPS: return 11;
145 case IB_RATE_50_GBPS: return 20;
146 case IB_RATE_400_GBPS: return 160;
147 case IB_RATE_600_GBPS: return 240;
148 default: return -1;
149 }
150}
151EXPORT_SYMBOL(ib_rate_to_mult);
152
153__attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
154{
155 switch (mult) {
156 case 1: return IB_RATE_2_5_GBPS;
157 case 2: return IB_RATE_5_GBPS;
158 case 4: return IB_RATE_10_GBPS;
159 case 8: return IB_RATE_20_GBPS;
160 case 12: return IB_RATE_30_GBPS;
161 case 16: return IB_RATE_40_GBPS;
162 case 24: return IB_RATE_60_GBPS;
163 case 32: return IB_RATE_80_GBPS;
164 case 48: return IB_RATE_120_GBPS;
165 case 6: return IB_RATE_14_GBPS;
166 case 22: return IB_RATE_56_GBPS;
167 case 45: return IB_RATE_112_GBPS;
168 case 67: return IB_RATE_168_GBPS;
169 case 10: return IB_RATE_25_GBPS;
170 case 40: return IB_RATE_100_GBPS;
171 case 80: return IB_RATE_200_GBPS;
172 case 120: return IB_RATE_300_GBPS;
173 case 11: return IB_RATE_28_GBPS;
174 case 20: return IB_RATE_50_GBPS;
175 case 160: return IB_RATE_400_GBPS;
176 case 240: return IB_RATE_600_GBPS;
177 default: return IB_RATE_PORT_CURRENT;
178 }
179}
180EXPORT_SYMBOL(mult_to_ib_rate);
181
182__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
183{
184 switch (rate) {
185 case IB_RATE_2_5_GBPS: return 2500;
186 case IB_RATE_5_GBPS: return 5000;
187 case IB_RATE_10_GBPS: return 10000;
188 case IB_RATE_20_GBPS: return 20000;
189 case IB_RATE_30_GBPS: return 30000;
190 case IB_RATE_40_GBPS: return 40000;
191 case IB_RATE_60_GBPS: return 60000;
192 case IB_RATE_80_GBPS: return 80000;
193 case IB_RATE_120_GBPS: return 120000;
194 case IB_RATE_14_GBPS: return 14062;
195 case IB_RATE_56_GBPS: return 56250;
196 case IB_RATE_112_GBPS: return 112500;
197 case IB_RATE_168_GBPS: return 168750;
198 case IB_RATE_25_GBPS: return 25781;
199 case IB_RATE_100_GBPS: return 103125;
200 case IB_RATE_200_GBPS: return 206250;
201 case IB_RATE_300_GBPS: return 309375;
202 case IB_RATE_28_GBPS: return 28125;
203 case IB_RATE_50_GBPS: return 53125;
204 case IB_RATE_400_GBPS: return 425000;
205 case IB_RATE_600_GBPS: return 637500;
206 default: return -1;
207 }
208}
209EXPORT_SYMBOL(ib_rate_to_mbps);
210
211__attribute_const__ enum rdma_transport_type
212rdma_node_get_transport(unsigned int node_type)
213{
214
215 if (node_type == RDMA_NODE_USNIC)
216 return RDMA_TRANSPORT_USNIC;
217 if (node_type == RDMA_NODE_USNIC_UDP)
218 return RDMA_TRANSPORT_USNIC_UDP;
219 if (node_type == RDMA_NODE_RNIC)
220 return RDMA_TRANSPORT_IWARP;
221 if (node_type == RDMA_NODE_UNSPECIFIED)
222 return RDMA_TRANSPORT_UNSPECIFIED;
223
224 return RDMA_TRANSPORT_IB;
225}
226EXPORT_SYMBOL(rdma_node_get_transport);
227
228enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
229{
230 enum rdma_transport_type lt;
231 if (device->ops.get_link_layer)
232 return device->ops.get_link_layer(device, port_num);
233
234 lt = rdma_node_get_transport(device->node_type);
235 if (lt == RDMA_TRANSPORT_IB)
236 return IB_LINK_LAYER_INFINIBAND;
237
238 return IB_LINK_LAYER_ETHERNET;
239}
240EXPORT_SYMBOL(rdma_port_get_link_layer);
241
242
243
244
245
246
247
248
249
250
251
252
253
254struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
255 const char *caller)
256{
257 struct ib_pd *pd;
258 int mr_access_flags = 0;
259 int ret;
260
261 pd = rdma_zalloc_drv_obj(device, ib_pd);
262 if (!pd)
263 return ERR_PTR(-ENOMEM);
264
265 pd->device = device;
266 pd->uobject = NULL;
267 pd->__internal_mr = NULL;
268 atomic_set(&pd->usecnt, 0);
269 pd->flags = flags;
270
271 pd->res.type = RDMA_RESTRACK_PD;
272 rdma_restrack_set_task(&pd->res, caller);
273
274 ret = device->ops.alloc_pd(pd, NULL);
275 if (ret) {
276 kfree(pd);
277 return ERR_PTR(ret);
278 }
279 rdma_restrack_kadd(&pd->res);
280
281 if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
282 pd->local_dma_lkey = device->local_dma_lkey;
283 else
284 mr_access_flags |= IB_ACCESS_LOCAL_WRITE;
285
286 if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
287 pr_warn("%s: enabling unsafe global rkey\n", caller);
288 mr_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE;
289 }
290
291 if (mr_access_flags) {
292 struct ib_mr *mr;
293
294 mr = pd->device->ops.get_dma_mr(pd, mr_access_flags);
295 if (IS_ERR(mr)) {
296 ib_dealloc_pd(pd);
297 return ERR_CAST(mr);
298 }
299
300 mr->device = pd->device;
301 mr->pd = pd;
302 mr->type = IB_MR_TYPE_DMA;
303 mr->uobject = NULL;
304 mr->need_inval = false;
305
306 pd->__internal_mr = mr;
307
308 if (!(device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY))
309 pd->local_dma_lkey = pd->__internal_mr->lkey;
310
311 if (flags & IB_PD_UNSAFE_GLOBAL_RKEY)
312 pd->unsafe_global_rkey = pd->__internal_mr->rkey;
313 }
314
315 return pd;
316}
317EXPORT_SYMBOL(__ib_alloc_pd);
318
319
320
321
322
323
324
325
326
327
328void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
329{
330 int ret;
331
332 if (pd->__internal_mr) {
333 ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL);
334 WARN_ON(ret);
335 pd->__internal_mr = NULL;
336 }
337
338
339
340 WARN_ON(atomic_read(&pd->usecnt));
341
342 rdma_restrack_del(&pd->res);
343 pd->device->ops.dealloc_pd(pd, udata);
344 kfree(pd);
345}
346EXPORT_SYMBOL(ib_dealloc_pd_user);
347
348
349
350
351
352
353
354
355
356void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
357 const struct rdma_ah_attr *src)
358{
359 *dest = *src;
360 if (dest->grh.sgid_attr)
361 rdma_hold_gid_attr(dest->grh.sgid_attr);
362}
363EXPORT_SYMBOL(rdma_copy_ah_attr);
364
365
366
367
368
369
370
371
372
373
374
375void rdma_replace_ah_attr(struct rdma_ah_attr *old,
376 const struct rdma_ah_attr *new)
377{
378 rdma_destroy_ah_attr(old);
379 *old = *new;
380 if (old->grh.sgid_attr)
381 rdma_hold_gid_attr(old->grh.sgid_attr);
382}
383EXPORT_SYMBOL(rdma_replace_ah_attr);
384
385
386
387
388
389
390
391
392
393
394
395
396void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src)
397{
398 rdma_destroy_ah_attr(dest);
399 *dest = *src;
400 src->grh.sgid_attr = NULL;
401}
402EXPORT_SYMBOL(rdma_move_ah_attr);
403
404
405
406
407
408static int rdma_check_ah_attr(struct ib_device *device,
409 struct rdma_ah_attr *ah_attr)
410{
411 if (!rdma_is_port_valid(device, ah_attr->port_num))
412 return -EINVAL;
413
414 if ((rdma_is_grh_required(device, ah_attr->port_num) ||
415 ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) &&
416 !(ah_attr->ah_flags & IB_AH_GRH))
417 return -EINVAL;
418
419 if (ah_attr->grh.sgid_attr) {
420
421
422
423
424 if (ah_attr->grh.sgid_attr->index != ah_attr->grh.sgid_index ||
425 ah_attr->grh.sgid_attr->port_num != ah_attr->port_num)
426 return -EINVAL;
427 }
428 return 0;
429}
430
431
432
433
434
435static int rdma_fill_sgid_attr(struct ib_device *device,
436 struct rdma_ah_attr *ah_attr,
437 const struct ib_gid_attr **old_sgid_attr)
438{
439 const struct ib_gid_attr *sgid_attr;
440 struct ib_global_route *grh;
441 int ret;
442
443 *old_sgid_attr = ah_attr->grh.sgid_attr;
444
445 ret = rdma_check_ah_attr(device, ah_attr);
446 if (ret)
447 return ret;
448
449 if (!(ah_attr->ah_flags & IB_AH_GRH))
450 return 0;
451
452 grh = rdma_ah_retrieve_grh(ah_attr);
453 if (grh->sgid_attr)
454 return 0;
455
456 sgid_attr =
457 rdma_get_gid_attr(device, ah_attr->port_num, grh->sgid_index);
458 if (IS_ERR(sgid_attr))
459 return PTR_ERR(sgid_attr);
460
461
462 grh->sgid_attr = sgid_attr;
463 return 0;
464}
465
466static void rdma_unfill_sgid_attr(struct rdma_ah_attr *ah_attr,
467 const struct ib_gid_attr *old_sgid_attr)
468{
469
470
471
472
473 if (ah_attr->grh.sgid_attr == old_sgid_attr)
474 return;
475
476
477
478
479
480
481 rdma_destroy_ah_attr(ah_attr);
482}
483
484static const struct ib_gid_attr *
485rdma_update_sgid_attr(struct rdma_ah_attr *ah_attr,
486 const struct ib_gid_attr *old_attr)
487{
488 if (old_attr)
489 rdma_put_gid_attr(old_attr);
490 if (ah_attr->ah_flags & IB_AH_GRH) {
491 rdma_hold_gid_attr(ah_attr->grh.sgid_attr);
492 return ah_attr->grh.sgid_attr;
493 }
494 return NULL;
495}
496
497static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
498 struct rdma_ah_attr *ah_attr,
499 u32 flags,
500 struct ib_udata *udata)
501{
502 struct ib_device *device = pd->device;
503 struct ib_ah *ah;
504 int ret;
505
506 might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
507
508 if (!device->ops.create_ah)
509 return ERR_PTR(-EOPNOTSUPP);
510
511 ah = rdma_zalloc_drv_obj_gfp(
512 device, ib_ah,
513 (flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
514 if (!ah)
515 return ERR_PTR(-ENOMEM);
516
517 ah->device = device;
518 ah->pd = pd;
519 ah->type = ah_attr->type;
520 ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
521
522 ret = device->ops.create_ah(ah, ah_attr, flags, udata);
523 if (ret) {
524 kfree(ah);
525 return ERR_PTR(ret);
526 }
527
528 atomic_inc(&pd->usecnt);
529 return ah;
530}
531
532
533
534
535
536
537
538
539
540
541
542
543struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
544 u32 flags)
545{
546 const struct ib_gid_attr *old_sgid_attr;
547 struct ib_ah *ah;
548 int ret;
549
550 ret = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
551 if (ret)
552 return ERR_PTR(ret);
553
554 ah = _rdma_create_ah(pd, ah_attr, flags, NULL);
555
556 rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
557 return ah;
558}
559EXPORT_SYMBOL(rdma_create_ah);
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
575 struct rdma_ah_attr *ah_attr,
576 struct ib_udata *udata)
577{
578 const struct ib_gid_attr *old_sgid_attr;
579 struct ib_ah *ah;
580 int err;
581
582 err = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
583 if (err)
584 return ERR_PTR(err);
585
586 if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
587 err = ib_resolve_eth_dmac(pd->device, ah_attr);
588 if (err) {
589 ah = ERR_PTR(err);
590 goto out;
591 }
592 }
593
594 ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE, udata);
595
596out:
597 rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
598 return ah;
599}
600EXPORT_SYMBOL(rdma_create_user_ah);
601
602int ib_get_rdma_header_version(const union rdma_network_hdr *hdr)
603{
604 const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh;
605 struct iphdr ip4h_checked;
606 const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh;
607
608
609
610
611 if (ip6h->version != 6)
612 return (ip4h->version == 4) ? 4 : 0;
613
614
615
616
617
618 if (ip4h->ihl != 5)
619 return 6;
620
621
622
623
624
625 memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked));
626 ip4h_checked.check = 0;
627 ip4h_checked.check = ip_fast_csum((u8 *)&ip4h_checked, 5);
628
629 if (ip4h->check == ip4h_checked.check)
630 return 4;
631 return 6;
632}
633EXPORT_SYMBOL(ib_get_rdma_header_version);
634
635static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
636 u8 port_num,
637 const struct ib_grh *grh)
638{
639 int grh_version;
640
641 if (rdma_protocol_ib(device, port_num))
642 return RDMA_NETWORK_IB;
643
644 grh_version = ib_get_rdma_header_version((union rdma_network_hdr *)grh);
645
646 if (grh_version == 4)
647 return RDMA_NETWORK_IPV4;
648
649 if (grh->next_hdr == IPPROTO_UDP)
650 return RDMA_NETWORK_IPV6;
651
652 return RDMA_NETWORK_ROCE_V1;
653}
654
655struct find_gid_index_context {
656 u16 vlan_id;
657 enum ib_gid_type gid_type;
658};
659
660static bool find_gid_index(const union ib_gid *gid,
661 const struct ib_gid_attr *gid_attr,
662 void *context)
663{
664 struct find_gid_index_context *ctx = context;
665 u16 vlan_id = 0xffff;
666 int ret;
667
668 if (ctx->gid_type != gid_attr->gid_type)
669 return false;
670
671 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
672 if (ret)
673 return false;
674
675 return ctx->vlan_id == vlan_id;
676}
677
678static const struct ib_gid_attr *
679get_sgid_attr_from_eth(struct ib_device *device, u8 port_num,
680 u16 vlan_id, const union ib_gid *sgid,
681 enum ib_gid_type gid_type)
682{
683 struct find_gid_index_context context = {.vlan_id = vlan_id,
684 .gid_type = gid_type};
685
686 return rdma_find_gid_by_filter(device, sgid, port_num, find_gid_index,
687 &context);
688}
689
690int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
691 enum rdma_network_type net_type,
692 union ib_gid *sgid, union ib_gid *dgid)
693{
694 struct sockaddr_in src_in;
695 struct sockaddr_in dst_in;
696 __be32 src_saddr, dst_saddr;
697
698 if (!sgid || !dgid)
699 return -EINVAL;
700
701 if (net_type == RDMA_NETWORK_IPV4) {
702 memcpy(&src_in.sin_addr.s_addr,
703 &hdr->roce4grh.saddr, 4);
704 memcpy(&dst_in.sin_addr.s_addr,
705 &hdr->roce4grh.daddr, 4);
706 src_saddr = src_in.sin_addr.s_addr;
707 dst_saddr = dst_in.sin_addr.s_addr;
708 ipv6_addr_set_v4mapped(src_saddr,
709 (struct in6_addr *)sgid);
710 ipv6_addr_set_v4mapped(dst_saddr,
711 (struct in6_addr *)dgid);
712 return 0;
713 } else if (net_type == RDMA_NETWORK_IPV6 ||
714 net_type == RDMA_NETWORK_IB) {
715 *dgid = hdr->ibgrh.dgid;
716 *sgid = hdr->ibgrh.sgid;
717 return 0;
718 } else {
719 return -EINVAL;
720 }
721}
722EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
723
724
725
726
727
728static int ib_resolve_unicast_gid_dmac(struct ib_device *device,
729 struct rdma_ah_attr *ah_attr)
730{
731 struct ib_global_route *grh = rdma_ah_retrieve_grh(ah_attr);
732 const struct ib_gid_attr *sgid_attr = grh->sgid_attr;
733 int hop_limit = 0xff;
734 int ret = 0;
735
736
737
738
739 if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) &&
740 sgid_attr->gid_type == IB_GID_TYPE_ROCE) {
741 rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
742 ah_attr->roce.dmac);
743 return ret;
744 }
745
746 ret = rdma_addr_find_l2_eth_by_grh(&sgid_attr->gid, &grh->dgid,
747 ah_attr->roce.dmac,
748 sgid_attr, &hop_limit);
749
750 grh->hop_limit = hop_limit;
751 return ret;
752}
753
754
755
756
757
758
759
760
761
762
763
764
765
766int ib_init_ah_attr_from_wc(struct ib_device *device, u8 port_num,
767 const struct ib_wc *wc, const struct ib_grh *grh,
768 struct rdma_ah_attr *ah_attr)
769{
770 u32 flow_class;
771 int ret;
772 enum rdma_network_type net_type = RDMA_NETWORK_IB;
773 enum ib_gid_type gid_type = IB_GID_TYPE_IB;
774 const struct ib_gid_attr *sgid_attr;
775 int hoplimit = 0xff;
776 union ib_gid dgid;
777 union ib_gid sgid;
778
779 might_sleep();
780
781 memset(ah_attr, 0, sizeof *ah_attr);
782 ah_attr->type = rdma_ah_find_type(device, port_num);
783 if (rdma_cap_eth_ah(device, port_num)) {
784 if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE)
785 net_type = wc->network_hdr_type;
786 else
787 net_type = ib_get_net_type_by_grh(device, port_num, grh);
788 gid_type = ib_network_to_gid_type(net_type);
789 }
790 ret = ib_get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
791 &sgid, &dgid);
792 if (ret)
793 return ret;
794
795 rdma_ah_set_sl(ah_attr, wc->sl);
796 rdma_ah_set_port_num(ah_attr, port_num);
797
798 if (rdma_protocol_roce(device, port_num)) {
799 u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
800 wc->vlan_id : 0xffff;
801
802 if (!(wc->wc_flags & IB_WC_GRH))
803 return -EPROTOTYPE;
804
805 sgid_attr = get_sgid_attr_from_eth(device, port_num,
806 vlan_id, &dgid,
807 gid_type);
808 if (IS_ERR(sgid_attr))
809 return PTR_ERR(sgid_attr);
810
811 flow_class = be32_to_cpu(grh->version_tclass_flow);
812 rdma_move_grh_sgid_attr(ah_attr,
813 &sgid,
814 flow_class & 0xFFFFF,
815 hoplimit,
816 (flow_class >> 20) & 0xFF,
817 sgid_attr);
818
819 ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
820 if (ret)
821 rdma_destroy_ah_attr(ah_attr);
822
823 return ret;
824 } else {
825 rdma_ah_set_dlid(ah_attr, wc->slid);
826 rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits);
827
828 if ((wc->wc_flags & IB_WC_GRH) == 0)
829 return 0;
830
831 if (dgid.global.interface_id !=
832 cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) {
833 sgid_attr = rdma_find_gid_by_port(
834 device, &dgid, IB_GID_TYPE_IB, port_num, NULL);
835 } else
836 sgid_attr = rdma_get_gid_attr(device, port_num, 0);
837
838 if (IS_ERR(sgid_attr))
839 return PTR_ERR(sgid_attr);
840 flow_class = be32_to_cpu(grh->version_tclass_flow);
841 rdma_move_grh_sgid_attr(ah_attr,
842 &sgid,
843 flow_class & 0xFFFFF,
844 hoplimit,
845 (flow_class >> 20) & 0xFF,
846 sgid_attr);
847
848 return 0;
849 }
850}
851EXPORT_SYMBOL(ib_init_ah_attr_from_wc);
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
869 u32 flow_label, u8 hop_limit, u8 traffic_class,
870 const struct ib_gid_attr *sgid_attr)
871{
872 rdma_ah_set_grh(attr, dgid, flow_label, sgid_attr->index, hop_limit,
873 traffic_class);
874 attr->grh.sgid_attr = sgid_attr;
875}
876EXPORT_SYMBOL(rdma_move_grh_sgid_attr);
877
878
879
880
881
882
883
884
885
886
887void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr)
888{
889 if (ah_attr->grh.sgid_attr) {
890 rdma_put_gid_attr(ah_attr->grh.sgid_attr);
891 ah_attr->grh.sgid_attr = NULL;
892 }
893}
894EXPORT_SYMBOL(rdma_destroy_ah_attr);
895
896struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
897 const struct ib_grh *grh, u8 port_num)
898{
899 struct rdma_ah_attr ah_attr;
900 struct ib_ah *ah;
901 int ret;
902
903 ret = ib_init_ah_attr_from_wc(pd->device, port_num, wc, grh, &ah_attr);
904 if (ret)
905 return ERR_PTR(ret);
906
907 ah = rdma_create_ah(pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
908
909 rdma_destroy_ah_attr(&ah_attr);
910 return ah;
911}
912EXPORT_SYMBOL(ib_create_ah_from_wc);
913
914int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
915{
916 const struct ib_gid_attr *old_sgid_attr;
917 int ret;
918
919 if (ah->type != ah_attr->type)
920 return -EINVAL;
921
922 ret = rdma_fill_sgid_attr(ah->device, ah_attr, &old_sgid_attr);
923 if (ret)
924 return ret;
925
926 ret = ah->device->ops.modify_ah ?
927 ah->device->ops.modify_ah(ah, ah_attr) :
928 -EOPNOTSUPP;
929
930 ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr);
931 rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
932 return ret;
933}
934EXPORT_SYMBOL(rdma_modify_ah);
935
936int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
937{
938 ah_attr->grh.sgid_attr = NULL;
939
940 return ah->device->ops.query_ah ?
941 ah->device->ops.query_ah(ah, ah_attr) :
942 -EOPNOTSUPP;
943}
944EXPORT_SYMBOL(rdma_query_ah);
945
946int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)
947{
948 const struct ib_gid_attr *sgid_attr = ah->sgid_attr;
949 struct ib_pd *pd;
950
951 might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
952
953 pd = ah->pd;
954
955 ah->device->ops.destroy_ah(ah, flags);
956 atomic_dec(&pd->usecnt);
957 if (sgid_attr)
958 rdma_put_gid_attr(sgid_attr);
959
960 kfree(ah);
961 return 0;
962}
963EXPORT_SYMBOL(rdma_destroy_ah_user);
964
965
966
967struct ib_srq *ib_create_srq(struct ib_pd *pd,
968 struct ib_srq_init_attr *srq_init_attr)
969{
970 struct ib_srq *srq;
971 int ret;
972
973 if (!pd->device->ops.create_srq)
974 return ERR_PTR(-EOPNOTSUPP);
975
976 srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
977 if (!srq)
978 return ERR_PTR(-ENOMEM);
979
980 srq->device = pd->device;
981 srq->pd = pd;
982 srq->event_handler = srq_init_attr->event_handler;
983 srq->srq_context = srq_init_attr->srq_context;
984 srq->srq_type = srq_init_attr->srq_type;
985
986 if (ib_srq_has_cq(srq->srq_type)) {
987 srq->ext.cq = srq_init_attr->ext.cq;
988 atomic_inc(&srq->ext.cq->usecnt);
989 }
990 if (srq->srq_type == IB_SRQT_XRC) {
991 srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
992 atomic_inc(&srq->ext.xrc.xrcd->usecnt);
993 }
994 atomic_inc(&pd->usecnt);
995
996 ret = pd->device->ops.create_srq(srq, srq_init_attr, NULL);
997 if (ret) {
998 atomic_dec(&srq->pd->usecnt);
999 if (srq->srq_type == IB_SRQT_XRC)
1000 atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1001 if (ib_srq_has_cq(srq->srq_type))
1002 atomic_dec(&srq->ext.cq->usecnt);
1003 kfree(srq);
1004 return ERR_PTR(ret);
1005 }
1006
1007 return srq;
1008}
1009EXPORT_SYMBOL(ib_create_srq);
1010
1011int ib_modify_srq(struct ib_srq *srq,
1012 struct ib_srq_attr *srq_attr,
1013 enum ib_srq_attr_mask srq_attr_mask)
1014{
1015 return srq->device->ops.modify_srq ?
1016 srq->device->ops.modify_srq(srq, srq_attr, srq_attr_mask,
1017 NULL) : -EOPNOTSUPP;
1018}
1019EXPORT_SYMBOL(ib_modify_srq);
1020
1021int ib_query_srq(struct ib_srq *srq,
1022 struct ib_srq_attr *srq_attr)
1023{
1024 return srq->device->ops.query_srq ?
1025 srq->device->ops.query_srq(srq, srq_attr) : -EOPNOTSUPP;
1026}
1027EXPORT_SYMBOL(ib_query_srq);
1028
1029int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)
1030{
1031 if (atomic_read(&srq->usecnt))
1032 return -EBUSY;
1033
1034 srq->device->ops.destroy_srq(srq, udata);
1035
1036 atomic_dec(&srq->pd->usecnt);
1037 if (srq->srq_type == IB_SRQT_XRC)
1038 atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1039 if (ib_srq_has_cq(srq->srq_type))
1040 atomic_dec(&srq->ext.cq->usecnt);
1041 kfree(srq);
1042
1043 return 0;
1044}
1045EXPORT_SYMBOL(ib_destroy_srq_user);
1046
1047
1048
1049static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
1050{
1051 struct ib_qp *qp = context;
1052 unsigned long flags;
1053
1054 spin_lock_irqsave(&qp->device->event_handler_lock, flags);
1055 list_for_each_entry(event->element.qp, &qp->open_list, open_list)
1056 if (event->element.qp->event_handler)
1057 event->element.qp->event_handler(event, event->element.qp->qp_context);
1058 spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
1059}
1060
1061static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
1062{
1063 mutex_lock(&xrcd->tgt_qp_mutex);
1064 list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
1065 mutex_unlock(&xrcd->tgt_qp_mutex);
1066}
1067
1068static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
1069 void (*event_handler)(struct ib_event *, void *),
1070 void *qp_context)
1071{
1072 struct ib_qp *qp;
1073 unsigned long flags;
1074 int err;
1075
1076 qp = kzalloc(sizeof *qp, GFP_KERNEL);
1077 if (!qp)
1078 return ERR_PTR(-ENOMEM);
1079
1080 qp->real_qp = real_qp;
1081 err = ib_open_shared_qp_security(qp, real_qp->device);
1082 if (err) {
1083 kfree(qp);
1084 return ERR_PTR(err);
1085 }
1086
1087 qp->real_qp = real_qp;
1088 atomic_inc(&real_qp->usecnt);
1089 qp->device = real_qp->device;
1090 qp->event_handler = event_handler;
1091 qp->qp_context = qp_context;
1092 qp->qp_num = real_qp->qp_num;
1093 qp->qp_type = real_qp->qp_type;
1094
1095 spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
1096 list_add(&qp->open_list, &real_qp->open_list);
1097 spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
1098
1099 return qp;
1100}
1101
1102struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
1103 struct ib_qp_open_attr *qp_open_attr)
1104{
1105 struct ib_qp *qp, *real_qp;
1106
1107 if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
1108 return ERR_PTR(-EINVAL);
1109
1110 qp = ERR_PTR(-EINVAL);
1111 mutex_lock(&xrcd->tgt_qp_mutex);
1112 list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
1113 if (real_qp->qp_num == qp_open_attr->qp_num) {
1114 qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
1115 qp_open_attr->qp_context);
1116 break;
1117 }
1118 }
1119 mutex_unlock(&xrcd->tgt_qp_mutex);
1120 return qp;
1121}
1122EXPORT_SYMBOL(ib_open_qp);
1123
1124static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
1125 struct ib_qp_init_attr *qp_init_attr,
1126 struct ib_udata *udata)
1127{
1128 struct ib_qp *real_qp = qp;
1129
1130 qp->event_handler = __ib_shared_qp_event_handler;
1131 qp->qp_context = qp;
1132 qp->pd = NULL;
1133 qp->send_cq = qp->recv_cq = NULL;
1134 qp->srq = NULL;
1135 qp->xrcd = qp_init_attr->xrcd;
1136 atomic_inc(&qp_init_attr->xrcd->usecnt);
1137 INIT_LIST_HEAD(&qp->open_list);
1138
1139 qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
1140 qp_init_attr->qp_context);
1141 if (IS_ERR(qp))
1142 return qp;
1143
1144 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
1145 return qp;
1146}
1147
1148struct ib_qp *ib_create_qp_user(struct ib_pd *pd,
1149 struct ib_qp_init_attr *qp_init_attr,
1150 struct ib_udata *udata)
1151{
1152 struct ib_device *device = pd ? pd->device : qp_init_attr->xrcd->device;
1153 struct ib_qp *qp;
1154 int ret;
1155
1156 if (qp_init_attr->rwq_ind_tbl &&
1157 (qp_init_attr->recv_cq ||
1158 qp_init_attr->srq || qp_init_attr->cap.max_recv_wr ||
1159 qp_init_attr->cap.max_recv_sge))
1160 return ERR_PTR(-EINVAL);
1161
1162 if ((qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) &&
1163 !(device->attrs.device_cap_flags & IB_DEVICE_INTEGRITY_HANDOVER))
1164 return ERR_PTR(-EINVAL);
1165
1166
1167
1168
1169
1170
1171
1172 if (qp_init_attr->cap.max_rdma_ctxs)
1173 rdma_rw_init_qp(device, qp_init_attr);
1174
1175 qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL);
1176 if (IS_ERR(qp))
1177 return qp;
1178
1179 ret = ib_create_qp_security(qp, device);
1180 if (ret)
1181 goto err;
1182
1183 qp->qp_type = qp_init_attr->qp_type;
1184 qp->rwq_ind_tbl = qp_init_attr->rwq_ind_tbl;
1185
1186 atomic_set(&qp->usecnt, 0);
1187 qp->mrs_used = 0;
1188 spin_lock_init(&qp->mr_lock);
1189 INIT_LIST_HEAD(&qp->rdma_mrs);
1190 INIT_LIST_HEAD(&qp->sig_mrs);
1191 qp->port = 0;
1192
1193 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
1194 struct ib_qp *xrc_qp =
1195 create_xrc_qp_user(qp, qp_init_attr, udata);
1196
1197 if (IS_ERR(xrc_qp)) {
1198 ret = PTR_ERR(xrc_qp);
1199 goto err;
1200 }
1201 return xrc_qp;
1202 }
1203
1204 qp->event_handler = qp_init_attr->event_handler;
1205 qp->qp_context = qp_init_attr->qp_context;
1206 if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
1207 qp->recv_cq = NULL;
1208 qp->srq = NULL;
1209 } else {
1210 qp->recv_cq = qp_init_attr->recv_cq;
1211 if (qp_init_attr->recv_cq)
1212 atomic_inc(&qp_init_attr->recv_cq->usecnt);
1213 qp->srq = qp_init_attr->srq;
1214 if (qp->srq)
1215 atomic_inc(&qp_init_attr->srq->usecnt);
1216 }
1217
1218 qp->send_cq = qp_init_attr->send_cq;
1219 qp->xrcd = NULL;
1220
1221 atomic_inc(&pd->usecnt);
1222 if (qp_init_attr->send_cq)
1223 atomic_inc(&qp_init_attr->send_cq->usecnt);
1224 if (qp_init_attr->rwq_ind_tbl)
1225 atomic_inc(&qp->rwq_ind_tbl->usecnt);
1226
1227 if (qp_init_attr->cap.max_rdma_ctxs) {
1228 ret = rdma_rw_init_mrs(qp, qp_init_attr);
1229 if (ret)
1230 goto err;
1231 }
1232
1233
1234
1235
1236
1237
1238 qp->max_write_sge = qp_init_attr->cap.max_send_sge;
1239 qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
1240 device->attrs.max_sge_rd);
1241 if (qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN)
1242 qp->integrity_en = true;
1243
1244 return qp;
1245
1246err:
1247 ib_destroy_qp(qp);
1248 return ERR_PTR(ret);
1249
1250}
1251EXPORT_SYMBOL(ib_create_qp_user);
1252
1253static const struct {
1254 int valid;
1255 enum ib_qp_attr_mask req_param[IB_QPT_MAX];
1256 enum ib_qp_attr_mask opt_param[IB_QPT_MAX];
1257} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
1258 [IB_QPS_RESET] = {
1259 [IB_QPS_RESET] = { .valid = 1 },
1260 [IB_QPS_INIT] = {
1261 .valid = 1,
1262 .req_param = {
1263 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1264 IB_QP_PORT |
1265 IB_QP_QKEY),
1266 [IB_QPT_RAW_PACKET] = IB_QP_PORT,
1267 [IB_QPT_UC] = (IB_QP_PKEY_INDEX |
1268 IB_QP_PORT |
1269 IB_QP_ACCESS_FLAGS),
1270 [IB_QPT_RC] = (IB_QP_PKEY_INDEX |
1271 IB_QP_PORT |
1272 IB_QP_ACCESS_FLAGS),
1273 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX |
1274 IB_QP_PORT |
1275 IB_QP_ACCESS_FLAGS),
1276 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX |
1277 IB_QP_PORT |
1278 IB_QP_ACCESS_FLAGS),
1279 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1280 IB_QP_QKEY),
1281 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1282 IB_QP_QKEY),
1283 }
1284 },
1285 },
1286 [IB_QPS_INIT] = {
1287 [IB_QPS_RESET] = { .valid = 1 },
1288 [IB_QPS_ERR] = { .valid = 1 },
1289 [IB_QPS_INIT] = {
1290 .valid = 1,
1291 .opt_param = {
1292 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1293 IB_QP_PORT |
1294 IB_QP_QKEY),
1295 [IB_QPT_UC] = (IB_QP_PKEY_INDEX |
1296 IB_QP_PORT |
1297 IB_QP_ACCESS_FLAGS),
1298 [IB_QPT_RC] = (IB_QP_PKEY_INDEX |
1299 IB_QP_PORT |
1300 IB_QP_ACCESS_FLAGS),
1301 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX |
1302 IB_QP_PORT |
1303 IB_QP_ACCESS_FLAGS),
1304 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX |
1305 IB_QP_PORT |
1306 IB_QP_ACCESS_FLAGS),
1307 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1308 IB_QP_QKEY),
1309 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1310 IB_QP_QKEY),
1311 }
1312 },
1313 [IB_QPS_RTR] = {
1314 .valid = 1,
1315 .req_param = {
1316 [IB_QPT_UC] = (IB_QP_AV |
1317 IB_QP_PATH_MTU |
1318 IB_QP_DEST_QPN |
1319 IB_QP_RQ_PSN),
1320 [IB_QPT_RC] = (IB_QP_AV |
1321 IB_QP_PATH_MTU |
1322 IB_QP_DEST_QPN |
1323 IB_QP_RQ_PSN |
1324 IB_QP_MAX_DEST_RD_ATOMIC |
1325 IB_QP_MIN_RNR_TIMER),
1326 [IB_QPT_XRC_INI] = (IB_QP_AV |
1327 IB_QP_PATH_MTU |
1328 IB_QP_DEST_QPN |
1329 IB_QP_RQ_PSN),
1330 [IB_QPT_XRC_TGT] = (IB_QP_AV |
1331 IB_QP_PATH_MTU |
1332 IB_QP_DEST_QPN |
1333 IB_QP_RQ_PSN |
1334 IB_QP_MAX_DEST_RD_ATOMIC |
1335 IB_QP_MIN_RNR_TIMER),
1336 },
1337 .opt_param = {
1338 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1339 IB_QP_QKEY),
1340 [IB_QPT_UC] = (IB_QP_ALT_PATH |
1341 IB_QP_ACCESS_FLAGS |
1342 IB_QP_PKEY_INDEX),
1343 [IB_QPT_RC] = (IB_QP_ALT_PATH |
1344 IB_QP_ACCESS_FLAGS |
1345 IB_QP_PKEY_INDEX),
1346 [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH |
1347 IB_QP_ACCESS_FLAGS |
1348 IB_QP_PKEY_INDEX),
1349 [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH |
1350 IB_QP_ACCESS_FLAGS |
1351 IB_QP_PKEY_INDEX),
1352 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1353 IB_QP_QKEY),
1354 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1355 IB_QP_QKEY),
1356 },
1357 },
1358 },
1359 [IB_QPS_RTR] = {
1360 [IB_QPS_RESET] = { .valid = 1 },
1361 [IB_QPS_ERR] = { .valid = 1 },
1362 [IB_QPS_RTS] = {
1363 .valid = 1,
1364 .req_param = {
1365 [IB_QPT_UD] = IB_QP_SQ_PSN,
1366 [IB_QPT_UC] = IB_QP_SQ_PSN,
1367 [IB_QPT_RC] = (IB_QP_TIMEOUT |
1368 IB_QP_RETRY_CNT |
1369 IB_QP_RNR_RETRY |
1370 IB_QP_SQ_PSN |
1371 IB_QP_MAX_QP_RD_ATOMIC),
1372 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT |
1373 IB_QP_RETRY_CNT |
1374 IB_QP_RNR_RETRY |
1375 IB_QP_SQ_PSN |
1376 IB_QP_MAX_QP_RD_ATOMIC),
1377 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT |
1378 IB_QP_SQ_PSN),
1379 [IB_QPT_SMI] = IB_QP_SQ_PSN,
1380 [IB_QPT_GSI] = IB_QP_SQ_PSN,
1381 },
1382 .opt_param = {
1383 [IB_QPT_UD] = (IB_QP_CUR_STATE |
1384 IB_QP_QKEY),
1385 [IB_QPT_UC] = (IB_QP_CUR_STATE |
1386 IB_QP_ALT_PATH |
1387 IB_QP_ACCESS_FLAGS |
1388 IB_QP_PATH_MIG_STATE),
1389 [IB_QPT_RC] = (IB_QP_CUR_STATE |
1390 IB_QP_ALT_PATH |
1391 IB_QP_ACCESS_FLAGS |
1392 IB_QP_MIN_RNR_TIMER |
1393 IB_QP_PATH_MIG_STATE),
1394 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
1395 IB_QP_ALT_PATH |
1396 IB_QP_ACCESS_FLAGS |
1397 IB_QP_PATH_MIG_STATE),
1398 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
1399 IB_QP_ALT_PATH |
1400 IB_QP_ACCESS_FLAGS |
1401 IB_QP_MIN_RNR_TIMER |
1402 IB_QP_PATH_MIG_STATE),
1403 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
1404 IB_QP_QKEY),
1405 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
1406 IB_QP_QKEY),
1407 [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1408 }
1409 }
1410 },
1411 [IB_QPS_RTS] = {
1412 [IB_QPS_RESET] = { .valid = 1 },
1413 [IB_QPS_ERR] = { .valid = 1 },
1414 [IB_QPS_RTS] = {
1415 .valid = 1,
1416 .opt_param = {
1417 [IB_QPT_UD] = (IB_QP_CUR_STATE |
1418 IB_QP_QKEY),
1419 [IB_QPT_UC] = (IB_QP_CUR_STATE |
1420 IB_QP_ACCESS_FLAGS |
1421 IB_QP_ALT_PATH |
1422 IB_QP_PATH_MIG_STATE),
1423 [IB_QPT_RC] = (IB_QP_CUR_STATE |
1424 IB_QP_ACCESS_FLAGS |
1425 IB_QP_ALT_PATH |
1426 IB_QP_PATH_MIG_STATE |
1427 IB_QP_MIN_RNR_TIMER),
1428 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
1429 IB_QP_ACCESS_FLAGS |
1430 IB_QP_ALT_PATH |
1431 IB_QP_PATH_MIG_STATE),
1432 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
1433 IB_QP_ACCESS_FLAGS |
1434 IB_QP_ALT_PATH |
1435 IB_QP_PATH_MIG_STATE |
1436 IB_QP_MIN_RNR_TIMER),
1437 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
1438 IB_QP_QKEY),
1439 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
1440 IB_QP_QKEY),
1441 [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1442 }
1443 },
1444 [IB_QPS_SQD] = {
1445 .valid = 1,
1446 .opt_param = {
1447 [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1448 [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1449 [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1450 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1451 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1452 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1453 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
1454 }
1455 },
1456 },
1457 [IB_QPS_SQD] = {
1458 [IB_QPS_RESET] = { .valid = 1 },
1459 [IB_QPS_ERR] = { .valid = 1 },
1460 [IB_QPS_RTS] = {
1461 .valid = 1,
1462 .opt_param = {
1463 [IB_QPT_UD] = (IB_QP_CUR_STATE |
1464 IB_QP_QKEY),
1465 [IB_QPT_UC] = (IB_QP_CUR_STATE |
1466 IB_QP_ALT_PATH |
1467 IB_QP_ACCESS_FLAGS |
1468 IB_QP_PATH_MIG_STATE),
1469 [IB_QPT_RC] = (IB_QP_CUR_STATE |
1470 IB_QP_ALT_PATH |
1471 IB_QP_ACCESS_FLAGS |
1472 IB_QP_MIN_RNR_TIMER |
1473 IB_QP_PATH_MIG_STATE),
1474 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE |
1475 IB_QP_ALT_PATH |
1476 IB_QP_ACCESS_FLAGS |
1477 IB_QP_PATH_MIG_STATE),
1478 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE |
1479 IB_QP_ALT_PATH |
1480 IB_QP_ACCESS_FLAGS |
1481 IB_QP_MIN_RNR_TIMER |
1482 IB_QP_PATH_MIG_STATE),
1483 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
1484 IB_QP_QKEY),
1485 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
1486 IB_QP_QKEY),
1487 }
1488 },
1489 [IB_QPS_SQD] = {
1490 .valid = 1,
1491 .opt_param = {
1492 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
1493 IB_QP_QKEY),
1494 [IB_QPT_UC] = (IB_QP_AV |
1495 IB_QP_ALT_PATH |
1496 IB_QP_ACCESS_FLAGS |
1497 IB_QP_PKEY_INDEX |
1498 IB_QP_PATH_MIG_STATE),
1499 [IB_QPT_RC] = (IB_QP_PORT |
1500 IB_QP_AV |
1501 IB_QP_TIMEOUT |
1502 IB_QP_RETRY_CNT |
1503 IB_QP_RNR_RETRY |
1504 IB_QP_MAX_QP_RD_ATOMIC |
1505 IB_QP_MAX_DEST_RD_ATOMIC |
1506 IB_QP_ALT_PATH |
1507 IB_QP_ACCESS_FLAGS |
1508 IB_QP_PKEY_INDEX |
1509 IB_QP_MIN_RNR_TIMER |
1510 IB_QP_PATH_MIG_STATE),
1511 [IB_QPT_XRC_INI] = (IB_QP_PORT |
1512 IB_QP_AV |
1513 IB_QP_TIMEOUT |
1514 IB_QP_RETRY_CNT |
1515 IB_QP_RNR_RETRY |
1516 IB_QP_MAX_QP_RD_ATOMIC |
1517 IB_QP_ALT_PATH |
1518 IB_QP_ACCESS_FLAGS |
1519 IB_QP_PKEY_INDEX |
1520 IB_QP_PATH_MIG_STATE),
1521 [IB_QPT_XRC_TGT] = (IB_QP_PORT |
1522 IB_QP_AV |
1523 IB_QP_TIMEOUT |
1524 IB_QP_MAX_DEST_RD_ATOMIC |
1525 IB_QP_ALT_PATH |
1526 IB_QP_ACCESS_FLAGS |
1527 IB_QP_PKEY_INDEX |
1528 IB_QP_MIN_RNR_TIMER |
1529 IB_QP_PATH_MIG_STATE),
1530 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
1531 IB_QP_QKEY),
1532 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
1533 IB_QP_QKEY),
1534 }
1535 }
1536 },
1537 [IB_QPS_SQE] = {
1538 [IB_QPS_RESET] = { .valid = 1 },
1539 [IB_QPS_ERR] = { .valid = 1 },
1540 [IB_QPS_RTS] = {
1541 .valid = 1,
1542 .opt_param = {
1543 [IB_QPT_UD] = (IB_QP_CUR_STATE |
1544 IB_QP_QKEY),
1545 [IB_QPT_UC] = (IB_QP_CUR_STATE |
1546 IB_QP_ACCESS_FLAGS),
1547 [IB_QPT_SMI] = (IB_QP_CUR_STATE |
1548 IB_QP_QKEY),
1549 [IB_QPT_GSI] = (IB_QP_CUR_STATE |
1550 IB_QP_QKEY),
1551 }
1552 }
1553 },
1554 [IB_QPS_ERR] = {
1555 [IB_QPS_RESET] = { .valid = 1 },
1556 [IB_QPS_ERR] = { .valid = 1 }
1557 }
1558};
1559
1560bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
1561 enum ib_qp_type type, enum ib_qp_attr_mask mask)
1562{
1563 enum ib_qp_attr_mask req_param, opt_param;
1564
1565 if (mask & IB_QP_CUR_STATE &&
1566 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
1567 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
1568 return false;
1569
1570 if (!qp_state_table[cur_state][next_state].valid)
1571 return false;
1572
1573 req_param = qp_state_table[cur_state][next_state].req_param[type];
1574 opt_param = qp_state_table[cur_state][next_state].opt_param[type];
1575
1576 if ((mask & req_param) != req_param)
1577 return false;
1578
1579 if (mask & ~(req_param | opt_param | IB_QP_STATE))
1580 return false;
1581
1582 return true;
1583}
1584EXPORT_SYMBOL(ib_modify_qp_is_ok);
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595static int ib_resolve_eth_dmac(struct ib_device *device,
1596 struct rdma_ah_attr *ah_attr)
1597{
1598 int ret = 0;
1599
1600 if (rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1601 if (ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1602 __be32 addr = 0;
1603
1604 memcpy(&addr, ah_attr->grh.dgid.raw + 12, 4);
1605 ip_eth_mc_map(addr, (char *)ah_attr->roce.dmac);
1606 } else {
1607 ipv6_eth_mc_map((struct in6_addr *)ah_attr->grh.dgid.raw,
1608 (char *)ah_attr->roce.dmac);
1609 }
1610 } else {
1611 ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
1612 }
1613 return ret;
1614}
1615
1616static bool is_qp_type_connected(const struct ib_qp *qp)
1617{
1618 return (qp->qp_type == IB_QPT_UC ||
1619 qp->qp_type == IB_QPT_RC ||
1620 qp->qp_type == IB_QPT_XRC_INI ||
1621 qp->qp_type == IB_QPT_XRC_TGT);
1622}
1623
1624
1625
1626
1627static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1628 int attr_mask, struct ib_udata *udata)
1629{
1630 u8 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1631 const struct ib_gid_attr *old_sgid_attr_av;
1632 const struct ib_gid_attr *old_sgid_attr_alt_av;
1633 int ret;
1634
1635 if (attr_mask & IB_QP_AV) {
1636 ret = rdma_fill_sgid_attr(qp->device, &attr->ah_attr,
1637 &old_sgid_attr_av);
1638 if (ret)
1639 return ret;
1640 }
1641 if (attr_mask & IB_QP_ALT_PATH) {
1642
1643
1644
1645
1646
1647
1648
1649 ret = rdma_fill_sgid_attr(qp->device, &attr->alt_ah_attr,
1650 &old_sgid_attr_alt_av);
1651 if (ret)
1652 goto out_av;
1653
1654
1655
1656
1657
1658 if (!(rdma_protocol_ib(qp->device,
1659 attr->alt_ah_attr.port_num) &&
1660 rdma_protocol_ib(qp->device, port))) {
1661 ret = EINVAL;
1662 goto out;
1663 }
1664 }
1665
1666
1667
1668
1669
1670 if (udata && (attr_mask & IB_QP_AV) &&
1671 attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
1672 is_qp_type_connected(qp)) {
1673 ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr);
1674 if (ret)
1675 goto out;
1676 }
1677
1678 if (rdma_ib_or_roce(qp->device, port)) {
1679 if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
1680 dev_warn(&qp->device->dev,
1681 "%s rq_psn overflow, masking to 24 bits\n",
1682 __func__);
1683 attr->rq_psn &= 0xffffff;
1684 }
1685
1686 if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
1687 dev_warn(&qp->device->dev,
1688 " %s sq_psn overflow, masking to 24 bits\n",
1689 __func__);
1690 attr->sq_psn &= 0xffffff;
1691 }
1692 }
1693
1694
1695
1696
1697
1698 if (!qp->counter && (attr_mask & IB_QP_PORT) &&
1699 ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_INIT))
1700 rdma_counter_bind_qp_auto(qp, attr->port_num);
1701
1702 ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
1703 if (ret)
1704 goto out;
1705
1706 if (attr_mask & IB_QP_PORT)
1707 qp->port = attr->port_num;
1708 if (attr_mask & IB_QP_AV)
1709 qp->av_sgid_attr =
1710 rdma_update_sgid_attr(&attr->ah_attr, qp->av_sgid_attr);
1711 if (attr_mask & IB_QP_ALT_PATH)
1712 qp->alt_path_sgid_attr = rdma_update_sgid_attr(
1713 &attr->alt_ah_attr, qp->alt_path_sgid_attr);
1714
1715out:
1716 if (attr_mask & IB_QP_ALT_PATH)
1717 rdma_unfill_sgid_attr(&attr->alt_ah_attr, old_sgid_attr_alt_av);
1718out_av:
1719 if (attr_mask & IB_QP_AV)
1720 rdma_unfill_sgid_attr(&attr->ah_attr, old_sgid_attr_av);
1721 return ret;
1722}
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,
1736 int attr_mask, struct ib_udata *udata)
1737{
1738 return _ib_modify_qp(ib_qp->real_qp, attr, attr_mask, udata);
1739}
1740EXPORT_SYMBOL(ib_modify_qp_with_udata);
1741
1742int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u8 *speed, u8 *width)
1743{
1744 int rc;
1745 u32 netdev_speed;
1746 struct net_device *netdev;
1747 struct ethtool_link_ksettings lksettings;
1748
1749 if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
1750 return -EINVAL;
1751
1752 netdev = ib_device_get_netdev(dev, port_num);
1753 if (!netdev)
1754 return -ENODEV;
1755
1756 rtnl_lock();
1757 rc = __ethtool_get_link_ksettings(netdev, &lksettings);
1758 rtnl_unlock();
1759
1760 dev_put(netdev);
1761
1762 if (!rc) {
1763 netdev_speed = lksettings.base.speed;
1764 } else {
1765 netdev_speed = SPEED_1000;
1766 pr_warn("%s speed is unknown, defaulting to %d\n", netdev->name,
1767 netdev_speed);
1768 }
1769
1770 if (netdev_speed <= SPEED_1000) {
1771 *width = IB_WIDTH_1X;
1772 *speed = IB_SPEED_SDR;
1773 } else if (netdev_speed <= SPEED_10000) {
1774 *width = IB_WIDTH_1X;
1775 *speed = IB_SPEED_FDR10;
1776 } else if (netdev_speed <= SPEED_20000) {
1777 *width = IB_WIDTH_4X;
1778 *speed = IB_SPEED_DDR;
1779 } else if (netdev_speed <= SPEED_25000) {
1780 *width = IB_WIDTH_1X;
1781 *speed = IB_SPEED_EDR;
1782 } else if (netdev_speed <= SPEED_40000) {
1783 *width = IB_WIDTH_4X;
1784 *speed = IB_SPEED_FDR10;
1785 } else {
1786 *width = IB_WIDTH_4X;
1787 *speed = IB_SPEED_EDR;
1788 }
1789
1790 return 0;
1791}
1792EXPORT_SYMBOL(ib_get_eth_speed);
1793
1794int ib_modify_qp(struct ib_qp *qp,
1795 struct ib_qp_attr *qp_attr,
1796 int qp_attr_mask)
1797{
1798 return _ib_modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
1799}
1800EXPORT_SYMBOL(ib_modify_qp);
1801
1802int ib_query_qp(struct ib_qp *qp,
1803 struct ib_qp_attr *qp_attr,
1804 int qp_attr_mask,
1805 struct ib_qp_init_attr *qp_init_attr)
1806{
1807 qp_attr->ah_attr.grh.sgid_attr = NULL;
1808 qp_attr->alt_ah_attr.grh.sgid_attr = NULL;
1809
1810 return qp->device->ops.query_qp ?
1811 qp->device->ops.query_qp(qp->real_qp, qp_attr, qp_attr_mask,
1812 qp_init_attr) : -EOPNOTSUPP;
1813}
1814EXPORT_SYMBOL(ib_query_qp);
1815
1816int ib_close_qp(struct ib_qp *qp)
1817{
1818 struct ib_qp *real_qp;
1819 unsigned long flags;
1820
1821 real_qp = qp->real_qp;
1822 if (real_qp == qp)
1823 return -EINVAL;
1824
1825 spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
1826 list_del(&qp->open_list);
1827 spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
1828
1829 atomic_dec(&real_qp->usecnt);
1830 if (qp->qp_sec)
1831 ib_close_shared_qp_security(qp->qp_sec);
1832 kfree(qp);
1833
1834 return 0;
1835}
1836EXPORT_SYMBOL(ib_close_qp);
1837
1838static int __ib_destroy_shared_qp(struct ib_qp *qp)
1839{
1840 struct ib_xrcd *xrcd;
1841 struct ib_qp *real_qp;
1842 int ret;
1843
1844 real_qp = qp->real_qp;
1845 xrcd = real_qp->xrcd;
1846
1847 mutex_lock(&xrcd->tgt_qp_mutex);
1848 ib_close_qp(qp);
1849 if (atomic_read(&real_qp->usecnt) == 0)
1850 list_del(&real_qp->xrcd_list);
1851 else
1852 real_qp = NULL;
1853 mutex_unlock(&xrcd->tgt_qp_mutex);
1854
1855 if (real_qp) {
1856 ret = ib_destroy_qp(real_qp);
1857 if (!ret)
1858 atomic_dec(&xrcd->usecnt);
1859 else
1860 __ib_insert_xrcd_qp(xrcd, real_qp);
1861 }
1862
1863 return 0;
1864}
1865
1866int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
1867{
1868 const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr;
1869 const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr;
1870 struct ib_pd *pd;
1871 struct ib_cq *scq, *rcq;
1872 struct ib_srq *srq;
1873 struct ib_rwq_ind_table *ind_tbl;
1874 struct ib_qp_security *sec;
1875 int ret;
1876
1877 WARN_ON_ONCE(qp->mrs_used > 0);
1878
1879 if (atomic_read(&qp->usecnt))
1880 return -EBUSY;
1881
1882 if (qp->real_qp != qp)
1883 return __ib_destroy_shared_qp(qp);
1884
1885 pd = qp->pd;
1886 scq = qp->send_cq;
1887 rcq = qp->recv_cq;
1888 srq = qp->srq;
1889 ind_tbl = qp->rwq_ind_tbl;
1890 sec = qp->qp_sec;
1891 if (sec)
1892 ib_destroy_qp_security_begin(sec);
1893
1894 if (!qp->uobject)
1895 rdma_rw_cleanup_mrs(qp);
1896
1897 rdma_counter_unbind_qp(qp, true);
1898 rdma_restrack_del(&qp->res);
1899 ret = qp->device->ops.destroy_qp(qp, udata);
1900 if (!ret) {
1901 if (alt_path_sgid_attr)
1902 rdma_put_gid_attr(alt_path_sgid_attr);
1903 if (av_sgid_attr)
1904 rdma_put_gid_attr(av_sgid_attr);
1905 if (pd)
1906 atomic_dec(&pd->usecnt);
1907 if (scq)
1908 atomic_dec(&scq->usecnt);
1909 if (rcq)
1910 atomic_dec(&rcq->usecnt);
1911 if (srq)
1912 atomic_dec(&srq->usecnt);
1913 if (ind_tbl)
1914 atomic_dec(&ind_tbl->usecnt);
1915 if (sec)
1916 ib_destroy_qp_security_end(sec);
1917 } else {
1918 if (sec)
1919 ib_destroy_qp_security_abort(sec);
1920 }
1921
1922 return ret;
1923}
1924EXPORT_SYMBOL(ib_destroy_qp_user);
1925
1926
1927
1928struct ib_cq *__ib_create_cq(struct ib_device *device,
1929 ib_comp_handler comp_handler,
1930 void (*event_handler)(struct ib_event *, void *),
1931 void *cq_context,
1932 const struct ib_cq_init_attr *cq_attr,
1933 const char *caller)
1934{
1935 struct ib_cq *cq;
1936 int ret;
1937
1938 cq = rdma_zalloc_drv_obj(device, ib_cq);
1939 if (!cq)
1940 return ERR_PTR(-ENOMEM);
1941
1942 cq->device = device;
1943 cq->uobject = NULL;
1944 cq->comp_handler = comp_handler;
1945 cq->event_handler = event_handler;
1946 cq->cq_context = cq_context;
1947 atomic_set(&cq->usecnt, 0);
1948 cq->res.type = RDMA_RESTRACK_CQ;
1949 rdma_restrack_set_task(&cq->res, caller);
1950
1951 ret = device->ops.create_cq(cq, cq_attr, NULL);
1952 if (ret) {
1953 kfree(cq);
1954 return ERR_PTR(ret);
1955 }
1956
1957 rdma_restrack_kadd(&cq->res);
1958 return cq;
1959}
1960EXPORT_SYMBOL(__ib_create_cq);
1961
1962int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
1963{
1964 return cq->device->ops.modify_cq ?
1965 cq->device->ops.modify_cq(cq, cq_count,
1966 cq_period) : -EOPNOTSUPP;
1967}
1968EXPORT_SYMBOL(rdma_set_cq_moderation);
1969
1970int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
1971{
1972 if (atomic_read(&cq->usecnt))
1973 return -EBUSY;
1974
1975 rdma_restrack_del(&cq->res);
1976 cq->device->ops.destroy_cq(cq, udata);
1977 kfree(cq);
1978 return 0;
1979}
1980EXPORT_SYMBOL(ib_destroy_cq_user);
1981
1982int ib_resize_cq(struct ib_cq *cq, int cqe)
1983{
1984 return cq->device->ops.resize_cq ?
1985 cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
1986}
1987EXPORT_SYMBOL(ib_resize_cq);
1988
1989
1990
1991int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
1992{
1993 struct ib_pd *pd = mr->pd;
1994 struct ib_dm *dm = mr->dm;
1995 struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
1996 int ret;
1997
1998 rdma_restrack_del(&mr->res);
1999 ret = mr->device->ops.dereg_mr(mr, udata);
2000 if (!ret) {
2001 atomic_dec(&pd->usecnt);
2002 if (dm)
2003 atomic_dec(&dm->usecnt);
2004 kfree(sig_attrs);
2005 }
2006
2007 return ret;
2008}
2009EXPORT_SYMBOL(ib_dereg_mr_user);
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
2025 u32 max_num_sg, struct ib_udata *udata)
2026{
2027 struct ib_mr *mr;
2028
2029 if (!pd->device->ops.alloc_mr)
2030 return ERR_PTR(-EOPNOTSUPP);
2031
2032 if (WARN_ON_ONCE(mr_type == IB_MR_TYPE_INTEGRITY))
2033 return ERR_PTR(-EINVAL);
2034
2035 mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);
2036 if (!IS_ERR(mr)) {
2037 mr->device = pd->device;
2038 mr->pd = pd;
2039 mr->dm = NULL;
2040 mr->uobject = NULL;
2041 atomic_inc(&pd->usecnt);
2042 mr->need_inval = false;
2043 mr->res.type = RDMA_RESTRACK_MR;
2044 rdma_restrack_kadd(&mr->res);
2045 mr->type = mr_type;
2046 mr->sig_attrs = NULL;
2047 }
2048
2049 return mr;
2050}
2051EXPORT_SYMBOL(ib_alloc_mr_user);
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
2066 u32 max_num_data_sg,
2067 u32 max_num_meta_sg)
2068{
2069 struct ib_mr *mr;
2070 struct ib_sig_attrs *sig_attrs;
2071
2072 if (!pd->device->ops.alloc_mr_integrity ||
2073 !pd->device->ops.map_mr_sg_pi)
2074 return ERR_PTR(-EOPNOTSUPP);
2075
2076 if (!max_num_meta_sg)
2077 return ERR_PTR(-EINVAL);
2078
2079 sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
2080 if (!sig_attrs)
2081 return ERR_PTR(-ENOMEM);
2082
2083 mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
2084 max_num_meta_sg);
2085 if (IS_ERR(mr)) {
2086 kfree(sig_attrs);
2087 return mr;
2088 }
2089
2090 mr->device = pd->device;
2091 mr->pd = pd;
2092 mr->dm = NULL;
2093 mr->uobject = NULL;
2094 atomic_inc(&pd->usecnt);
2095 mr->need_inval = false;
2096 mr->res.type = RDMA_RESTRACK_MR;
2097 rdma_restrack_kadd(&mr->res);
2098 mr->type = IB_MR_TYPE_INTEGRITY;
2099 mr->sig_attrs = sig_attrs;
2100
2101 return mr;
2102}
2103EXPORT_SYMBOL(ib_alloc_mr_integrity);
2104
2105
2106
2107struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
2108 int mr_access_flags,
2109 struct ib_fmr_attr *fmr_attr)
2110{
2111 struct ib_fmr *fmr;
2112
2113 if (!pd->device->ops.alloc_fmr)
2114 return ERR_PTR(-EOPNOTSUPP);
2115
2116 fmr = pd->device->ops.alloc_fmr(pd, mr_access_flags, fmr_attr);
2117 if (!IS_ERR(fmr)) {
2118 fmr->device = pd->device;
2119 fmr->pd = pd;
2120 atomic_inc(&pd->usecnt);
2121 }
2122
2123 return fmr;
2124}
2125EXPORT_SYMBOL(ib_alloc_fmr);
2126
2127int ib_unmap_fmr(struct list_head *fmr_list)
2128{
2129 struct ib_fmr *fmr;
2130
2131 if (list_empty(fmr_list))
2132 return 0;
2133
2134 fmr = list_entry(fmr_list->next, struct ib_fmr, list);
2135 return fmr->device->ops.unmap_fmr(fmr_list);
2136}
2137EXPORT_SYMBOL(ib_unmap_fmr);
2138
2139int ib_dealloc_fmr(struct ib_fmr *fmr)
2140{
2141 struct ib_pd *pd;
2142 int ret;
2143
2144 pd = fmr->pd;
2145 ret = fmr->device->ops.dealloc_fmr(fmr);
2146 if (!ret)
2147 atomic_dec(&pd->usecnt);
2148
2149 return ret;
2150}
2151EXPORT_SYMBOL(ib_dealloc_fmr);
2152
2153
2154
2155static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
2156{
2157 struct ib_qp_init_attr init_attr = {};
2158 struct ib_qp_attr attr = {};
2159 int num_eth_ports = 0;
2160 int port;
2161
2162
2163
2164
2165 if (!ib_query_qp(qp, &attr, IB_QP_STATE | IB_QP_PORT, &init_attr)) {
2166 if (attr.qp_state >= IB_QPS_INIT) {
2167 if (rdma_port_get_link_layer(qp->device, attr.port_num) !=
2168 IB_LINK_LAYER_INFINIBAND)
2169 return true;
2170 goto lid_check;
2171 }
2172 }
2173
2174
2175 for (port = 0; port < qp->device->phys_port_cnt; port++)
2176 if (rdma_port_get_link_layer(qp->device, port) !=
2177 IB_LINK_LAYER_INFINIBAND)
2178 num_eth_ports++;
2179
2180
2181
2182
2183
2184 if (num_eth_ports)
2185 return true;
2186
2187
2188lid_check:
2189 return !(lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
2190 lid == be16_to_cpu(IB_LID_PERMISSIVE));
2191}
2192
2193int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2194{
2195 int ret;
2196
2197 if (!qp->device->ops.attach_mcast)
2198 return -EOPNOTSUPP;
2199
2200 if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2201 qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2202 return -EINVAL;
2203
2204 ret = qp->device->ops.attach_mcast(qp, gid, lid);
2205 if (!ret)
2206 atomic_inc(&qp->usecnt);
2207 return ret;
2208}
2209EXPORT_SYMBOL(ib_attach_mcast);
2210
2211int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2212{
2213 int ret;
2214
2215 if (!qp->device->ops.detach_mcast)
2216 return -EOPNOTSUPP;
2217
2218 if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2219 qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2220 return -EINVAL;
2221
2222 ret = qp->device->ops.detach_mcast(qp, gid, lid);
2223 if (!ret)
2224 atomic_dec(&qp->usecnt);
2225 return ret;
2226}
2227EXPORT_SYMBOL(ib_detach_mcast);
2228
2229struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller)
2230{
2231 struct ib_xrcd *xrcd;
2232
2233 if (!device->ops.alloc_xrcd)
2234 return ERR_PTR(-EOPNOTSUPP);
2235
2236 xrcd = device->ops.alloc_xrcd(device, NULL);
2237 if (!IS_ERR(xrcd)) {
2238 xrcd->device = device;
2239 xrcd->inode = NULL;
2240 atomic_set(&xrcd->usecnt, 0);
2241 mutex_init(&xrcd->tgt_qp_mutex);
2242 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
2243 }
2244
2245 return xrcd;
2246}
2247EXPORT_SYMBOL(__ib_alloc_xrcd);
2248
2249int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
2250{
2251 struct ib_qp *qp;
2252 int ret;
2253
2254 if (atomic_read(&xrcd->usecnt))
2255 return -EBUSY;
2256
2257 while (!list_empty(&xrcd->tgt_qp_list)) {
2258 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
2259 ret = ib_destroy_qp(qp);
2260 if (ret)
2261 return ret;
2262 }
2263 mutex_destroy(&xrcd->tgt_qp_mutex);
2264
2265 return xrcd->device->ops.dealloc_xrcd(xrcd, udata);
2266}
2267EXPORT_SYMBOL(ib_dealloc_xrcd);
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283struct ib_wq *ib_create_wq(struct ib_pd *pd,
2284 struct ib_wq_init_attr *wq_attr)
2285{
2286 struct ib_wq *wq;
2287
2288 if (!pd->device->ops.create_wq)
2289 return ERR_PTR(-EOPNOTSUPP);
2290
2291 wq = pd->device->ops.create_wq(pd, wq_attr, NULL);
2292 if (!IS_ERR(wq)) {
2293 wq->event_handler = wq_attr->event_handler;
2294 wq->wq_context = wq_attr->wq_context;
2295 wq->wq_type = wq_attr->wq_type;
2296 wq->cq = wq_attr->cq;
2297 wq->device = pd->device;
2298 wq->pd = pd;
2299 wq->uobject = NULL;
2300 atomic_inc(&pd->usecnt);
2301 atomic_inc(&wq_attr->cq->usecnt);
2302 atomic_set(&wq->usecnt, 0);
2303 }
2304 return wq;
2305}
2306EXPORT_SYMBOL(ib_create_wq);
2307
2308
2309
2310
2311
2312
2313int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
2314{
2315 struct ib_cq *cq = wq->cq;
2316 struct ib_pd *pd = wq->pd;
2317
2318 if (atomic_read(&wq->usecnt))
2319 return -EBUSY;
2320
2321 wq->device->ops.destroy_wq(wq, udata);
2322 atomic_dec(&pd->usecnt);
2323 atomic_dec(&cq->usecnt);
2324
2325 return 0;
2326}
2327EXPORT_SYMBOL(ib_destroy_wq);
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
2338 u32 wq_attr_mask)
2339{
2340 int err;
2341
2342 if (!wq->device->ops.modify_wq)
2343 return -EOPNOTSUPP;
2344
2345 err = wq->device->ops.modify_wq(wq, wq_attr, wq_attr_mask, NULL);
2346 return err;
2347}
2348EXPORT_SYMBOL(ib_modify_wq);
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360struct ib_rwq_ind_table *ib_create_rwq_ind_table(struct ib_device *device,
2361 struct ib_rwq_ind_table_init_attr *init_attr)
2362{
2363 struct ib_rwq_ind_table *rwq_ind_table;
2364 int i;
2365 u32 table_size;
2366
2367 if (!device->ops.create_rwq_ind_table)
2368 return ERR_PTR(-EOPNOTSUPP);
2369
2370 table_size = (1 << init_attr->log_ind_tbl_size);
2371 rwq_ind_table = device->ops.create_rwq_ind_table(device,
2372 init_attr, NULL);
2373 if (IS_ERR(rwq_ind_table))
2374 return rwq_ind_table;
2375
2376 rwq_ind_table->ind_tbl = init_attr->ind_tbl;
2377 rwq_ind_table->log_ind_tbl_size = init_attr->log_ind_tbl_size;
2378 rwq_ind_table->device = device;
2379 rwq_ind_table->uobject = NULL;
2380 atomic_set(&rwq_ind_table->usecnt, 0);
2381
2382 for (i = 0; i < table_size; i++)
2383 atomic_inc(&rwq_ind_table->ind_tbl[i]->usecnt);
2384
2385 return rwq_ind_table;
2386}
2387EXPORT_SYMBOL(ib_create_rwq_ind_table);
2388
2389
2390
2391
2392
2393int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table)
2394{
2395 int err, i;
2396 u32 table_size = (1 << rwq_ind_table->log_ind_tbl_size);
2397 struct ib_wq **ind_tbl = rwq_ind_table->ind_tbl;
2398
2399 if (atomic_read(&rwq_ind_table->usecnt))
2400 return -EBUSY;
2401
2402 err = rwq_ind_table->device->ops.destroy_rwq_ind_table(rwq_ind_table);
2403 if (!err) {
2404 for (i = 0; i < table_size; i++)
2405 atomic_dec(&ind_tbl[i]->usecnt);
2406 }
2407
2408 return err;
2409}
2410EXPORT_SYMBOL(ib_destroy_rwq_ind_table);
2411
2412int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
2413 struct ib_mr_status *mr_status)
2414{
2415 if (!mr->device->ops.check_mr_status)
2416 return -EOPNOTSUPP;
2417
2418 return mr->device->ops.check_mr_status(mr, check_mask, mr_status);
2419}
2420EXPORT_SYMBOL(ib_check_mr_status);
2421
2422int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,
2423 int state)
2424{
2425 if (!device->ops.set_vf_link_state)
2426 return -EOPNOTSUPP;
2427
2428 return device->ops.set_vf_link_state(device, vf, port, state);
2429}
2430EXPORT_SYMBOL(ib_set_vf_link_state);
2431
2432int ib_get_vf_config(struct ib_device *device, int vf, u8 port,
2433 struct ifla_vf_info *info)
2434{
2435 if (!device->ops.get_vf_config)
2436 return -EOPNOTSUPP;
2437
2438 return device->ops.get_vf_config(device, vf, port, info);
2439}
2440EXPORT_SYMBOL(ib_get_vf_config);
2441
2442int ib_get_vf_stats(struct ib_device *device, int vf, u8 port,
2443 struct ifla_vf_stats *stats)
2444{
2445 if (!device->ops.get_vf_stats)
2446 return -EOPNOTSUPP;
2447
2448 return device->ops.get_vf_stats(device, vf, port, stats);
2449}
2450EXPORT_SYMBOL(ib_get_vf_stats);
2451
2452int ib_set_vf_guid(struct ib_device *device, int vf, u8 port, u64 guid,
2453 int type)
2454{
2455 if (!device->ops.set_vf_guid)
2456 return -EOPNOTSUPP;
2457
2458 return device->ops.set_vf_guid(device, vf, port, guid, type);
2459}
2460EXPORT_SYMBOL(ib_set_vf_guid);
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
2483 int data_sg_nents, unsigned int *data_sg_offset,
2484 struct scatterlist *meta_sg, int meta_sg_nents,
2485 unsigned int *meta_sg_offset, unsigned int page_size)
2486{
2487 if (unlikely(!mr->device->ops.map_mr_sg_pi ||
2488 WARN_ON_ONCE(mr->type != IB_MR_TYPE_INTEGRITY)))
2489 return -EOPNOTSUPP;
2490
2491 mr->page_size = page_size;
2492
2493 return mr->device->ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,
2494 data_sg_offset, meta_sg,
2495 meta_sg_nents, meta_sg_offset);
2496}
2497EXPORT_SYMBOL(ib_map_mr_sg_pi);
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2525 unsigned int *sg_offset, unsigned int page_size)
2526{
2527 if (unlikely(!mr->device->ops.map_mr_sg))
2528 return -EOPNOTSUPP;
2529
2530 mr->page_size = page_size;
2531
2532 return mr->device->ops.map_mr_sg(mr, sg, sg_nents, sg_offset);
2533}
2534EXPORT_SYMBOL(ib_map_mr_sg);
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
2557 unsigned int *sg_offset_p, int (*set_page)(struct ib_mr *, u64))
2558{
2559 struct scatterlist *sg;
2560 u64 last_end_dma_addr = 0;
2561 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2562 unsigned int last_page_off = 0;
2563 u64 page_mask = ~((u64)mr->page_size - 1);
2564 int i, ret;
2565
2566 if (unlikely(sg_nents <= 0 || sg_offset > sg_dma_len(&sgl[0])))
2567 return -EINVAL;
2568
2569 mr->iova = sg_dma_address(&sgl[0]) + sg_offset;
2570 mr->length = 0;
2571
2572 for_each_sg(sgl, sg, sg_nents, i) {
2573 u64 dma_addr = sg_dma_address(sg) + sg_offset;
2574 u64 prev_addr = dma_addr;
2575 unsigned int dma_len = sg_dma_len(sg) - sg_offset;
2576 u64 end_dma_addr = dma_addr + dma_len;
2577 u64 page_addr = dma_addr & page_mask;
2578
2579
2580
2581
2582
2583
2584 if (i && (last_page_off != 0 || page_addr != dma_addr)) {
2585
2586 if (last_end_dma_addr != dma_addr)
2587 break;
2588
2589
2590
2591
2592
2593
2594 goto next_page;
2595 }
2596
2597 do {
2598 ret = set_page(mr, page_addr);
2599 if (unlikely(ret < 0)) {
2600 sg_offset = prev_addr - sg_dma_address(sg);
2601 mr->length += prev_addr - dma_addr;
2602 if (sg_offset_p)
2603 *sg_offset_p = sg_offset;
2604 return i || sg_offset ? i : ret;
2605 }
2606 prev_addr = page_addr;
2607next_page:
2608 page_addr += mr->page_size;
2609 } while (page_addr < end_dma_addr);
2610
2611 mr->length += dma_len;
2612 last_end_dma_addr = end_dma_addr;
2613 last_page_off = end_dma_addr & ~page_mask;
2614
2615 sg_offset = 0;
2616 }
2617
2618 if (sg_offset_p)
2619 *sg_offset_p = 0;
2620 return i;
2621}
2622EXPORT_SYMBOL(ib_sg_to_pages);
2623
2624struct ib_drain_cqe {
2625 struct ib_cqe cqe;
2626 struct completion done;
2627};
2628
2629static void ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
2630{
2631 struct ib_drain_cqe *cqe = container_of(wc->wr_cqe, struct ib_drain_cqe,
2632 cqe);
2633
2634 complete(&cqe->done);
2635}
2636
2637
2638
2639
2640static void __ib_drain_sq(struct ib_qp *qp)
2641{
2642 struct ib_cq *cq = qp->send_cq;
2643 struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2644 struct ib_drain_cqe sdrain;
2645 struct ib_rdma_wr swr = {
2646 .wr = {
2647 .next = NULL,
2648 { .wr_cqe = &sdrain.cqe, },
2649 .opcode = IB_WR_RDMA_WRITE,
2650 },
2651 };
2652 int ret;
2653
2654 ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2655 if (ret) {
2656 WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2657 return;
2658 }
2659
2660 sdrain.cqe.done = ib_drain_qp_done;
2661 init_completion(&sdrain.done);
2662
2663 ret = ib_post_send(qp, &swr.wr, NULL);
2664 if (ret) {
2665 WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2666 return;
2667 }
2668
2669 if (cq->poll_ctx == IB_POLL_DIRECT)
2670 while (wait_for_completion_timeout(&sdrain.done, HZ / 10) <= 0)
2671 ib_process_cq_direct(cq, -1);
2672 else
2673 wait_for_completion(&sdrain.done);
2674}
2675
2676
2677
2678
2679static void __ib_drain_rq(struct ib_qp *qp)
2680{
2681 struct ib_cq *cq = qp->recv_cq;
2682 struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2683 struct ib_drain_cqe rdrain;
2684 struct ib_recv_wr rwr = {};
2685 int ret;
2686
2687 ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2688 if (ret) {
2689 WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2690 return;
2691 }
2692
2693 rwr.wr_cqe = &rdrain.cqe;
2694 rdrain.cqe.done = ib_drain_qp_done;
2695 init_completion(&rdrain.done);
2696
2697 ret = ib_post_recv(qp, &rwr, NULL);
2698 if (ret) {
2699 WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2700 return;
2701 }
2702
2703 if (cq->poll_ctx == IB_POLL_DIRECT)
2704 while (wait_for_completion_timeout(&rdrain.done, HZ / 10) <= 0)
2705 ib_process_cq_direct(cq, -1);
2706 else
2707 wait_for_completion(&rdrain.done);
2708}
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729void ib_drain_sq(struct ib_qp *qp)
2730{
2731 if (qp->device->ops.drain_sq)
2732 qp->device->ops.drain_sq(qp);
2733 else
2734 __ib_drain_sq(qp);
2735}
2736EXPORT_SYMBOL(ib_drain_sq);
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757void ib_drain_rq(struct ib_qp *qp)
2758{
2759 if (qp->device->ops.drain_rq)
2760 qp->device->ops.drain_rq(qp);
2761 else
2762 __ib_drain_rq(qp);
2763}
2764EXPORT_SYMBOL(ib_drain_rq);
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781void ib_drain_qp(struct ib_qp *qp)
2782{
2783 ib_drain_sq(qp);
2784 if (!qp->srq)
2785 ib_drain_rq(qp);
2786}
2787EXPORT_SYMBOL(ib_drain_qp);
2788
2789struct net_device *rdma_alloc_netdev(struct ib_device *device, u8 port_num,
2790 enum rdma_netdev_t type, const char *name,
2791 unsigned char name_assign_type,
2792 void (*setup)(struct net_device *))
2793{
2794 struct rdma_netdev_alloc_params params;
2795 struct net_device *netdev;
2796 int rc;
2797
2798 if (!device->ops.rdma_netdev_get_params)
2799 return ERR_PTR(-EOPNOTSUPP);
2800
2801 rc = device->ops.rdma_netdev_get_params(device, port_num, type,
2802 ¶ms);
2803 if (rc)
2804 return ERR_PTR(rc);
2805
2806 netdev = alloc_netdev_mqs(params.sizeof_priv, name, name_assign_type,
2807 setup, params.txqs, params.rxqs);
2808 if (!netdev)
2809 return ERR_PTR(-ENOMEM);
2810
2811 return netdev;
2812}
2813EXPORT_SYMBOL(rdma_alloc_netdev);
2814
2815int rdma_init_netdev(struct ib_device *device, u8 port_num,
2816 enum rdma_netdev_t type, const char *name,
2817 unsigned char name_assign_type,
2818 void (*setup)(struct net_device *),
2819 struct net_device *netdev)
2820{
2821 struct rdma_netdev_alloc_params params;
2822 int rc;
2823
2824 if (!device->ops.rdma_netdev_get_params)
2825 return -EOPNOTSUPP;
2826
2827 rc = device->ops.rdma_netdev_get_params(device, port_num, type,
2828 ¶ms);
2829 if (rc)
2830 return rc;
2831
2832 return params.initialize_rdma_netdev(device, port_num,
2833 netdev, params.param);
2834}
2835EXPORT_SYMBOL(rdma_init_netdev);
2836
2837void __rdma_block_iter_start(struct ib_block_iter *biter,
2838 struct scatterlist *sglist, unsigned int nents,
2839 unsigned long pgsz)
2840{
2841 memset(biter, 0, sizeof(struct ib_block_iter));
2842 biter->__sg = sglist;
2843 biter->__sg_nents = nents;
2844
2845
2846 biter->__pg_bit = __fls(pgsz);
2847}
2848EXPORT_SYMBOL(__rdma_block_iter_start);
2849
2850bool __rdma_block_iter_next(struct ib_block_iter *biter)
2851{
2852 unsigned int block_offset;
2853
2854 if (!biter->__sg_nents || !biter->__sg)
2855 return false;
2856
2857 biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
2858 block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
2859 biter->__sg_advance += BIT_ULL(biter->__pg_bit) - block_offset;
2860
2861 if (biter->__sg_advance >= sg_dma_len(biter->__sg)) {
2862 biter->__sg_advance = 0;
2863 biter->__sg = sg_next(biter->__sg);
2864 biter->__sg_nents--;
2865 }
2866
2867 return true;
2868}
2869EXPORT_SYMBOL(__rdma_block_iter_next);
2870