1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/acpi.h>
34#include <linux/of_platform.h>
35#include <linux/module.h>
36#include <linux/pci.h>
37#include <rdma/ib_addr.h>
38#include <rdma/ib_smi.h>
39#include <rdma/ib_user_verbs.h>
40#include <rdma/ib_cache.h>
41#include "hns_roce_common.h"
42#include "hns_roce_device.h"
43#include "hns_roce_hem.h"
44
45static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u32 port,
46 const u8 *addr)
47{
48 u8 phy_port;
49 u32 i;
50
51 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
52 return 0;
53
54 if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN))
55 return 0;
56
57 for (i = 0; i < ETH_ALEN; i++)
58 hr_dev->dev_addr[port][i] = addr[i];
59
60 phy_port = hr_dev->iboe.phy_port[port];
61 return hr_dev->hw->set_mac(hr_dev, phy_port, addr);
62}
63
64static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context)
65{
66 struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
67 u32 port = attr->port_num - 1;
68 int ret;
69
70 if (port >= hr_dev->caps.num_ports)
71 return -EINVAL;
72
73 ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr);
74
75 return ret;
76}
77
78static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
79{
80 struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
81 u32 port = attr->port_num - 1;
82 int ret;
83
84 if (port >= hr_dev->caps.num_ports)
85 return -EINVAL;
86
87 ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, NULL, NULL);
88
89 return ret;
90}
91
92static int handle_en_event(struct hns_roce_dev *hr_dev, u32 port,
93 unsigned long event)
94{
95 struct device *dev = hr_dev->dev;
96 struct net_device *netdev;
97 int ret = 0;
98
99 netdev = hr_dev->iboe.netdevs[port];
100 if (!netdev) {
101 dev_err(dev, "Can't find netdev on port(%u)!\n", port);
102 return -ENODEV;
103 }
104
105 switch (event) {
106 case NETDEV_UP:
107 case NETDEV_CHANGE:
108 case NETDEV_REGISTER:
109 case NETDEV_CHANGEADDR:
110 ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
111 break;
112 case NETDEV_DOWN:
113
114
115
116 break;
117 default:
118 dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event));
119 break;
120 }
121
122 return ret;
123}
124
125static int hns_roce_netdev_event(struct notifier_block *self,
126 unsigned long event, void *ptr)
127{
128 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
129 struct hns_roce_ib_iboe *iboe = NULL;
130 struct hns_roce_dev *hr_dev = NULL;
131 int ret;
132 u32 port;
133
134 hr_dev = container_of(self, struct hns_roce_dev, iboe.nb);
135 iboe = &hr_dev->iboe;
136
137 for (port = 0; port < hr_dev->caps.num_ports; port++) {
138 if (dev == iboe->netdevs[port]) {
139 ret = handle_en_event(hr_dev, port, event);
140 if (ret)
141 return NOTIFY_DONE;
142 break;
143 }
144 }
145
146 return NOTIFY_DONE;
147}
148
149static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev)
150{
151 int ret;
152 u8 i;
153
154 for (i = 0; i < hr_dev->caps.num_ports; i++) {
155 if (hr_dev->hw->set_mtu)
156 hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i],
157 hr_dev->caps.max_mtu);
158 ret = hns_roce_set_mac(hr_dev, i,
159 hr_dev->iboe.netdevs[i]->dev_addr);
160 if (ret)
161 return ret;
162 }
163
164 return 0;
165}
166
167static int hns_roce_query_device(struct ib_device *ib_dev,
168 struct ib_device_attr *props,
169 struct ib_udata *uhw)
170{
171 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
172
173 memset(props, 0, sizeof(*props));
174
175 props->fw_ver = hr_dev->caps.fw_ver;
176 props->sys_image_guid = cpu_to_be64(hr_dev->sys_image_guid);
177 props->max_mr_size = (u64)(~(0ULL));
178 props->page_size_cap = hr_dev->caps.page_size_cap;
179 props->vendor_id = hr_dev->vendor_id;
180 props->vendor_part_id = hr_dev->vendor_part_id;
181 props->hw_ver = hr_dev->hw_rev;
182 props->max_qp = hr_dev->caps.num_qps;
183 props->max_qp_wr = hr_dev->caps.max_wqes;
184 props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT |
185 IB_DEVICE_RC_RNR_NAK_GEN;
186 props->max_send_sge = hr_dev->caps.max_sq_sg;
187 props->max_recv_sge = hr_dev->caps.max_rq_sg;
188 props->max_sge_rd = 1;
189 props->max_cq = hr_dev->caps.num_cqs;
190 props->max_cqe = hr_dev->caps.max_cqes;
191 props->max_mr = hr_dev->caps.num_mtpts;
192 props->max_pd = hr_dev->caps.num_pds;
193 props->max_qp_rd_atom = hr_dev->caps.max_qp_dest_rdma;
194 props->max_qp_init_rd_atom = hr_dev->caps.max_qp_init_rdma;
195 props->atomic_cap = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_ATOMIC ?
196 IB_ATOMIC_HCA : IB_ATOMIC_NONE;
197 props->max_pkeys = 1;
198 props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay;
199 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
200 props->max_srq = hr_dev->caps.num_srqs;
201 props->max_srq_wr = hr_dev->caps.max_srq_wrs;
202 props->max_srq_sge = hr_dev->caps.max_srq_sges;
203 }
204
205 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR &&
206 hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
207 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
208 props->max_fast_reg_page_list_len = HNS_ROCE_FRMR_MAX_PA;
209 }
210
211 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)
212 props->device_cap_flags |= IB_DEVICE_XRC;
213
214 return 0;
215}
216
217static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num,
218 struct ib_port_attr *props)
219{
220 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
221 struct device *dev = hr_dev->dev;
222 struct net_device *net_dev;
223 unsigned long flags;
224 enum ib_mtu mtu;
225 u32 port;
226
227 port = port_num - 1;
228
229
230
231 props->max_mtu = hr_dev->caps.max_mtu;
232 props->gid_tbl_len = hr_dev->caps.gid_table_len[port];
233 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
234 IB_PORT_VENDOR_CLASS_SUP |
235 IB_PORT_BOOT_MGMT_SUP;
236 props->max_msg_sz = HNS_ROCE_MAX_MSG_LEN;
237 props->pkey_tbl_len = 1;
238 props->active_width = IB_WIDTH_4X;
239 props->active_speed = 1;
240
241 spin_lock_irqsave(&hr_dev->iboe.lock, flags);
242
243 net_dev = hr_dev->iboe.netdevs[port];
244 if (!net_dev) {
245 spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
246 dev_err(dev, "Find netdev %u failed!\n", port);
247 return -EINVAL;
248 }
249
250 mtu = iboe_get_mtu(net_dev->mtu);
251 props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256;
252 props->state = netif_running(net_dev) && netif_carrier_ok(net_dev) ?
253 IB_PORT_ACTIVE :
254 IB_PORT_DOWN;
255 props->phys_state = props->state == IB_PORT_ACTIVE ?
256 IB_PORT_PHYS_STATE_LINK_UP :
257 IB_PORT_PHYS_STATE_DISABLED;
258
259 spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
260
261 return 0;
262}
263
264static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device,
265 u32 port_num)
266{
267 return IB_LINK_LAYER_ETHERNET;
268}
269
270static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index,
271 u16 *pkey)
272{
273 *pkey = PKEY_ID;
274
275 return 0;
276}
277
278static int hns_roce_modify_device(struct ib_device *ib_dev, int mask,
279 struct ib_device_modify *props)
280{
281 unsigned long flags;
282
283 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
284 return -EOPNOTSUPP;
285
286 if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
287 spin_lock_irqsave(&to_hr_dev(ib_dev)->sm_lock, flags);
288 memcpy(ib_dev->node_desc, props->node_desc, NODE_DESC_SIZE);
289 spin_unlock_irqrestore(&to_hr_dev(ib_dev)->sm_lock, flags);
290 }
291
292 return 0;
293}
294
295struct hns_user_mmap_entry *
296hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
297 size_t length,
298 enum hns_roce_mmap_type mmap_type)
299{
300 struct hns_user_mmap_entry *entry;
301 int ret;
302
303 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
304 if (!entry)
305 return NULL;
306
307 entry->address = address;
308 entry->mmap_type = mmap_type;
309
310 ret = rdma_user_mmap_entry_insert_exact(
311 ucontext, &entry->rdma_entry, length,
312 mmap_type == HNS_ROCE_MMAP_TYPE_DB ? 0 : 1);
313 if (ret) {
314 kfree(entry);
315 return NULL;
316 }
317
318 return entry;
319}
320
321static void hns_roce_dealloc_uar_entry(struct hns_roce_ucontext *context)
322{
323 if (context->db_mmap_entry)
324 rdma_user_mmap_entry_remove(
325 &context->db_mmap_entry->rdma_entry);
326
327 if (context->tptr_mmap_entry)
328 rdma_user_mmap_entry_remove(
329 &context->tptr_mmap_entry->rdma_entry);
330}
331
332static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx)
333{
334 struct hns_roce_ucontext *context = to_hr_ucontext(uctx);
335 struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
336 u64 address;
337 int ret;
338
339 address = context->uar.pfn << PAGE_SHIFT;
340 context->db_mmap_entry = hns_roce_user_mmap_entry_insert(
341 uctx, address, PAGE_SIZE, HNS_ROCE_MMAP_TYPE_DB);
342 if (!context->db_mmap_entry)
343 return -ENOMEM;
344
345 if (!hr_dev->tptr_dma_addr || !hr_dev->tptr_size)
346 return 0;
347
348
349
350
351
352 context->tptr_mmap_entry =
353 hns_roce_user_mmap_entry_insert(uctx, hr_dev->tptr_dma_addr,
354 hr_dev->tptr_size,
355 HNS_ROCE_MMAP_TYPE_TPTR);
356 if (!context->tptr_mmap_entry) {
357 ret = -ENOMEM;
358 goto err;
359 }
360
361 return 0;
362
363err:
364 hns_roce_dealloc_uar_entry(context);
365 return ret;
366}
367
368static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
369 struct ib_udata *udata)
370{
371 int ret;
372 struct hns_roce_ucontext *context = to_hr_ucontext(uctx);
373 struct hns_roce_ib_alloc_ucontext_resp resp = {};
374 struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
375
376 if (!hr_dev->active)
377 return -EAGAIN;
378
379 resp.qp_tab_size = hr_dev->caps.num_qps;
380 resp.srq_tab_size = hr_dev->caps.num_srqs;
381
382 ret = hns_roce_uar_alloc(hr_dev, &context->uar);
383 if (ret)
384 goto error_fail_uar_alloc;
385
386 ret = hns_roce_alloc_uar_entry(uctx);
387 if (ret)
388 goto error_fail_uar_entry;
389
390 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||
391 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) {
392 INIT_LIST_HEAD(&context->page_list);
393 mutex_init(&context->page_mutex);
394 }
395
396 resp.cqe_size = hr_dev->caps.cqe_sz;
397
398 ret = ib_copy_to_udata(udata, &resp,
399 min(udata->outlen, sizeof(resp)));
400 if (ret)
401 goto error_fail_copy_to_udata;
402
403 return 0;
404
405error_fail_copy_to_udata:
406 hns_roce_dealloc_uar_entry(context);
407
408error_fail_uar_entry:
409 ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx);
410
411error_fail_uar_alloc:
412 return ret;
413}
414
415static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext)
416{
417 struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext);
418 struct hns_roce_dev *hr_dev = to_hr_dev(ibcontext->device);
419
420 hns_roce_dealloc_uar_entry(context);
421
422 ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx);
423}
424
425static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma)
426{
427 struct rdma_user_mmap_entry *rdma_entry;
428 struct hns_user_mmap_entry *entry;
429 phys_addr_t pfn;
430 pgprot_t prot;
431 int ret;
432
433 rdma_entry = rdma_user_mmap_entry_get_pgoff(uctx, vma->vm_pgoff);
434 if (!rdma_entry)
435 return -EINVAL;
436
437 entry = to_hns_mmap(rdma_entry);
438 pfn = entry->address >> PAGE_SHIFT;
439 prot = vma->vm_page_prot;
440
441 if (entry->mmap_type != HNS_ROCE_MMAP_TYPE_TPTR)
442 prot = pgprot_noncached(prot);
443
444 ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE,
445 prot, rdma_entry);
446
447 rdma_user_mmap_entry_put(rdma_entry);
448
449 return ret;
450}
451
452static void hns_roce_free_mmap(struct rdma_user_mmap_entry *rdma_entry)
453{
454 struct hns_user_mmap_entry *entry = to_hns_mmap(rdma_entry);
455
456 kfree(entry);
457}
458
459static int hns_roce_port_immutable(struct ib_device *ib_dev, u32 port_num,
460 struct ib_port_immutable *immutable)
461{
462 struct ib_port_attr attr;
463 int ret;
464
465 ret = ib_query_port(ib_dev, port_num, &attr);
466 if (ret)
467 return ret;
468
469 immutable->pkey_tbl_len = attr.pkey_tbl_len;
470 immutable->gid_tbl_len = attr.gid_tbl_len;
471
472 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
473 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
474 if (to_hr_dev(ib_dev)->caps.flags & HNS_ROCE_CAP_FLAG_ROCE_V1_V2)
475 immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
476
477 return 0;
478}
479
480static void hns_roce_disassociate_ucontext(struct ib_ucontext *ibcontext)
481{
482}
483
484static void hns_roce_get_fw_ver(struct ib_device *device, char *str)
485{
486 u64 fw_ver = to_hr_dev(device)->caps.fw_ver;
487 unsigned int major, minor, sub_minor;
488
489 major = upper_32_bits(fw_ver);
490 minor = high_16_bits(lower_32_bits(fw_ver));
491 sub_minor = low_16_bits(fw_ver);
492
493 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%04u", major, minor,
494 sub_minor);
495}
496
497static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)
498{
499 struct hns_roce_ib_iboe *iboe = &hr_dev->iboe;
500
501 hr_dev->active = false;
502 unregister_netdevice_notifier(&iboe->nb);
503 ib_unregister_device(&hr_dev->ib_dev);
504}
505
506static const struct ib_device_ops hns_roce_dev_ops = {
507 .owner = THIS_MODULE,
508 .driver_id = RDMA_DRIVER_HNS,
509 .uverbs_abi_ver = 1,
510 .uverbs_no_driver_id_binding = 1,
511
512 .get_dev_fw_str = hns_roce_get_fw_ver,
513 .add_gid = hns_roce_add_gid,
514 .alloc_pd = hns_roce_alloc_pd,
515 .alloc_ucontext = hns_roce_alloc_ucontext,
516 .create_ah = hns_roce_create_ah,
517 .create_user_ah = hns_roce_create_ah,
518 .create_cq = hns_roce_create_cq,
519 .create_qp = hns_roce_create_qp,
520 .dealloc_pd = hns_roce_dealloc_pd,
521 .dealloc_ucontext = hns_roce_dealloc_ucontext,
522 .del_gid = hns_roce_del_gid,
523 .dereg_mr = hns_roce_dereg_mr,
524 .destroy_ah = hns_roce_destroy_ah,
525 .destroy_cq = hns_roce_destroy_cq,
526 .disassociate_ucontext = hns_roce_disassociate_ucontext,
527 .fill_res_cq_entry = hns_roce_fill_res_cq_entry,
528 .get_dma_mr = hns_roce_get_dma_mr,
529 .get_link_layer = hns_roce_get_link_layer,
530 .get_port_immutable = hns_roce_port_immutable,
531 .mmap = hns_roce_mmap,
532 .mmap_free = hns_roce_free_mmap,
533 .modify_device = hns_roce_modify_device,
534 .modify_qp = hns_roce_modify_qp,
535 .query_ah = hns_roce_query_ah,
536 .query_device = hns_roce_query_device,
537 .query_pkey = hns_roce_query_pkey,
538 .query_port = hns_roce_query_port,
539 .reg_user_mr = hns_roce_reg_user_mr,
540
541 INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah),
542 INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq),
543 INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd),
544 INIT_RDMA_OBJ_SIZE(ib_qp, hns_roce_qp, ibqp),
545 INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext),
546};
547
548static const struct ib_device_ops hns_roce_dev_mr_ops = {
549 .rereg_user_mr = hns_roce_rereg_user_mr,
550};
551
552static const struct ib_device_ops hns_roce_dev_mw_ops = {
553 .alloc_mw = hns_roce_alloc_mw,
554 .dealloc_mw = hns_roce_dealloc_mw,
555
556 INIT_RDMA_OBJ_SIZE(ib_mw, hns_roce_mw, ibmw),
557};
558
559static const struct ib_device_ops hns_roce_dev_frmr_ops = {
560 .alloc_mr = hns_roce_alloc_mr,
561 .map_mr_sg = hns_roce_map_mr_sg,
562};
563
564static const struct ib_device_ops hns_roce_dev_srq_ops = {
565 .create_srq = hns_roce_create_srq,
566 .destroy_srq = hns_roce_destroy_srq,
567
568 INIT_RDMA_OBJ_SIZE(ib_srq, hns_roce_srq, ibsrq),
569};
570
571static const struct ib_device_ops hns_roce_dev_xrcd_ops = {
572 .alloc_xrcd = hns_roce_alloc_xrcd,
573 .dealloc_xrcd = hns_roce_dealloc_xrcd,
574
575 INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd),
576};
577
578static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
579{
580 int ret;
581 struct hns_roce_ib_iboe *iboe = NULL;
582 struct ib_device *ib_dev = NULL;
583 struct device *dev = hr_dev->dev;
584 unsigned int i;
585
586 iboe = &hr_dev->iboe;
587 spin_lock_init(&iboe->lock);
588
589 ib_dev = &hr_dev->ib_dev;
590
591 ib_dev->node_type = RDMA_NODE_IB_CA;
592 ib_dev->dev.parent = dev;
593
594 ib_dev->phys_port_cnt = hr_dev->caps.num_ports;
595 ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey;
596 ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors;
597
598 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR)
599 ib_set_device_ops(ib_dev, &hns_roce_dev_mr_ops);
600
601 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW)
602 ib_set_device_ops(ib_dev, &hns_roce_dev_mw_ops);
603
604 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR)
605 ib_set_device_ops(ib_dev, &hns_roce_dev_frmr_ops);
606
607 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
608 ib_set_device_ops(ib_dev, &hns_roce_dev_srq_ops);
609 ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops);
610 }
611
612 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)
613 ib_set_device_ops(ib_dev, &hns_roce_dev_xrcd_ops);
614
615 ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops);
616 ib_set_device_ops(ib_dev, &hns_roce_dev_ops);
617 for (i = 0; i < hr_dev->caps.num_ports; i++) {
618 if (!hr_dev->iboe.netdevs[i])
619 continue;
620
621 ret = ib_device_set_netdev(ib_dev, hr_dev->iboe.netdevs[i],
622 i + 1);
623 if (ret)
624 return ret;
625 }
626 dma_set_max_seg_size(dev, UINT_MAX);
627 ret = ib_register_device(ib_dev, "hns_%d", dev);
628 if (ret) {
629 dev_err(dev, "ib_register_device failed!\n");
630 return ret;
631 }
632
633 ret = hns_roce_setup_mtu_mac(hr_dev);
634 if (ret) {
635 dev_err(dev, "setup_mtu_mac failed!\n");
636 goto error_failed_setup_mtu_mac;
637 }
638
639 iboe->nb.notifier_call = hns_roce_netdev_event;
640 ret = register_netdevice_notifier(&iboe->nb);
641 if (ret) {
642 dev_err(dev, "register_netdevice_notifier failed!\n");
643 goto error_failed_setup_mtu_mac;
644 }
645
646 hr_dev->active = true;
647 return 0;
648
649error_failed_setup_mtu_mac:
650 ib_unregister_device(ib_dev);
651
652 return ret;
653}
654
655static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
656{
657 struct device *dev = hr_dev->dev;
658 int ret;
659
660 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table,
661 HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz,
662 hr_dev->caps.num_mtpts, 1);
663 if (ret) {
664 dev_err(dev, "Failed to init MTPT context memory, aborting.\n");
665 return ret;
666 }
667
668 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table,
669 HEM_TYPE_QPC, hr_dev->caps.qpc_sz,
670 hr_dev->caps.num_qps, 1);
671 if (ret) {
672 dev_err(dev, "Failed to init QP context memory, aborting.\n");
673 goto err_unmap_dmpt;
674 }
675
676 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.irrl_table,
677 HEM_TYPE_IRRL,
678 hr_dev->caps.irrl_entry_sz *
679 hr_dev->caps.max_qp_init_rdma,
680 hr_dev->caps.num_qps, 1);
681 if (ret) {
682 dev_err(dev, "Failed to init irrl_table memory, aborting.\n");
683 goto err_unmap_qp;
684 }
685
686 if (hr_dev->caps.trrl_entry_sz) {
687 ret = hns_roce_init_hem_table(hr_dev,
688 &hr_dev->qp_table.trrl_table,
689 HEM_TYPE_TRRL,
690 hr_dev->caps.trrl_entry_sz *
691 hr_dev->caps.max_qp_dest_rdma,
692 hr_dev->caps.num_qps, 1);
693 if (ret) {
694 dev_err(dev,
695 "Failed to init trrl_table memory, aborting.\n");
696 goto err_unmap_irrl;
697 }
698 }
699
700 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table,
701 HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz,
702 hr_dev->caps.num_cqs, 1);
703 if (ret) {
704 dev_err(dev, "Failed to init CQ context memory, aborting.\n");
705 goto err_unmap_trrl;
706 }
707
708 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
709 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->srq_table.table,
710 HEM_TYPE_SRQC,
711 hr_dev->caps.srqc_entry_sz,
712 hr_dev->caps.num_srqs, 1);
713 if (ret) {
714 dev_err(dev,
715 "Failed to init SRQ context memory, aborting.\n");
716 goto err_unmap_cq;
717 }
718 }
719
720 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
721 ret = hns_roce_init_hem_table(hr_dev,
722 &hr_dev->qp_table.sccc_table,
723 HEM_TYPE_SCCC,
724 hr_dev->caps.sccc_sz,
725 hr_dev->caps.num_qps, 1);
726 if (ret) {
727 dev_err(dev,
728 "Failed to init SCC context memory, aborting.\n");
729 goto err_unmap_srq;
730 }
731 }
732
733 if (hr_dev->caps.qpc_timer_entry_sz) {
734 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qpc_timer_table,
735 HEM_TYPE_QPC_TIMER,
736 hr_dev->caps.qpc_timer_entry_sz,
737 hr_dev->caps.num_qpc_timer, 1);
738 if (ret) {
739 dev_err(dev,
740 "Failed to init QPC timer memory, aborting.\n");
741 goto err_unmap_ctx;
742 }
743 }
744
745 if (hr_dev->caps.cqc_timer_entry_sz) {
746 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cqc_timer_table,
747 HEM_TYPE_CQC_TIMER,
748 hr_dev->caps.cqc_timer_entry_sz,
749 hr_dev->caps.num_cqc_timer, 1);
750 if (ret) {
751 dev_err(dev,
752 "Failed to init CQC timer memory, aborting.\n");
753 goto err_unmap_qpc_timer;
754 }
755 }
756
757 if (hr_dev->caps.gmv_entry_sz) {
758 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->gmv_table,
759 HEM_TYPE_GMV,
760 hr_dev->caps.gmv_entry_sz,
761 hr_dev->caps.gmv_entry_num, 1);
762 if (ret) {
763 dev_err(dev,
764 "failed to init gmv table memory, ret = %d\n",
765 ret);
766 goto err_unmap_cqc_timer;
767 }
768 }
769
770 return 0;
771
772err_unmap_cqc_timer:
773 if (hr_dev->caps.cqc_timer_entry_sz)
774 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cqc_timer_table);
775
776err_unmap_qpc_timer:
777 if (hr_dev->caps.qpc_timer_entry_sz)
778 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qpc_timer_table);
779
780err_unmap_ctx:
781 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
782 hns_roce_cleanup_hem_table(hr_dev,
783 &hr_dev->qp_table.sccc_table);
784err_unmap_srq:
785 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
786 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table);
787
788err_unmap_cq:
789 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
790
791err_unmap_trrl:
792 if (hr_dev->caps.trrl_entry_sz)
793 hns_roce_cleanup_hem_table(hr_dev,
794 &hr_dev->qp_table.trrl_table);
795
796err_unmap_irrl:
797 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
798
799err_unmap_qp:
800 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
801
802err_unmap_dmpt:
803 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
804
805 return ret;
806}
807
808
809
810
811
812
813static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)
814{
815 struct device *dev = hr_dev->dev;
816 int ret;
817
818 spin_lock_init(&hr_dev->sm_lock);
819 spin_lock_init(&hr_dev->bt_cmd_lock);
820
821 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||
822 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) {
823 INIT_LIST_HEAD(&hr_dev->pgdir_list);
824 mutex_init(&hr_dev->pgdir_mutex);
825 }
826
827 hns_roce_init_uar_table(hr_dev);
828
829 ret = hns_roce_uar_alloc(hr_dev, &hr_dev->priv_uar);
830 if (ret) {
831 dev_err(dev, "Failed to allocate priv_uar.\n");
832 goto err_uar_table_free;
833 }
834
835 ret = hns_roce_init_qp_table(hr_dev);
836 if (ret) {
837 dev_err(dev, "Failed to init qp_table.\n");
838 goto err_uar_table_free;
839 }
840
841 hns_roce_init_pd_table(hr_dev);
842
843 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)
844 hns_roce_init_xrcd_table(hr_dev);
845
846 hns_roce_init_mr_table(hr_dev);
847
848 hns_roce_init_cq_table(hr_dev);
849
850 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
851 hns_roce_init_srq_table(hr_dev);
852 }
853
854 return 0;
855
856err_uar_table_free:
857 ida_destroy(&hr_dev->uar_ida.ida);
858 return ret;
859}
860
861static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq)
862{
863 struct hns_roce_cq *hr_cq = to_hr_cq(cq);
864 unsigned long flags;
865
866 spin_lock_irqsave(&hr_cq->lock, flags);
867 if (cq->comp_handler) {
868 if (!hr_cq->is_armed) {
869 hr_cq->is_armed = 1;
870 list_add_tail(&hr_cq->node, cq_list);
871 }
872 }
873 spin_unlock_irqrestore(&hr_cq->lock, flags);
874}
875
876void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev)
877{
878 struct hns_roce_qp *hr_qp;
879 struct hns_roce_cq *hr_cq;
880 struct list_head cq_list;
881 unsigned long flags_qp;
882 unsigned long flags;
883
884 INIT_LIST_HEAD(&cq_list);
885
886 spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
887 list_for_each_entry(hr_qp, &hr_dev->qp_list, node) {
888 spin_lock_irqsave(&hr_qp->sq.lock, flags_qp);
889 if (hr_qp->sq.tail != hr_qp->sq.head)
890 check_and_get_armed_cq(&cq_list, hr_qp->ibqp.send_cq);
891 spin_unlock_irqrestore(&hr_qp->sq.lock, flags_qp);
892
893 spin_lock_irqsave(&hr_qp->rq.lock, flags_qp);
894 if ((!hr_qp->ibqp.srq) && (hr_qp->rq.tail != hr_qp->rq.head))
895 check_and_get_armed_cq(&cq_list, hr_qp->ibqp.recv_cq);
896 spin_unlock_irqrestore(&hr_qp->rq.lock, flags_qp);
897 }
898
899 list_for_each_entry(hr_cq, &cq_list, node)
900 hns_roce_cq_completion(hr_dev, hr_cq->cqn);
901
902 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
903}
904
905int hns_roce_init(struct hns_roce_dev *hr_dev)
906{
907 struct device *dev = hr_dev->dev;
908 int ret;
909
910 if (hr_dev->hw->reset) {
911 ret = hr_dev->hw->reset(hr_dev, true);
912 if (ret) {
913 dev_err(dev, "Reset RoCE engine failed!\n");
914 return ret;
915 }
916 }
917 hr_dev->is_reset = false;
918
919 if (hr_dev->hw->cmq_init) {
920 ret = hr_dev->hw->cmq_init(hr_dev);
921 if (ret) {
922 dev_err(dev, "Init RoCE Command Queue failed!\n");
923 goto error_failed_cmq_init;
924 }
925 }
926
927 ret = hr_dev->hw->hw_profile(hr_dev);
928 if (ret) {
929 dev_err(dev, "Get RoCE engine profile failed!\n");
930 goto error_failed_cmd_init;
931 }
932
933 ret = hns_roce_cmd_init(hr_dev);
934 if (ret) {
935 dev_err(dev, "cmd init failed!\n");
936 goto error_failed_cmd_init;
937 }
938
939
940 ret = hr_dev->hw->init_eq(hr_dev);
941 if (ret) {
942 dev_err(dev, "eq init failed!\n");
943 goto error_failed_eq_table;
944 }
945
946 if (hr_dev->cmd_mod) {
947 ret = hns_roce_cmd_use_events(hr_dev);
948 if (ret)
949 dev_warn(dev,
950 "Cmd event mode failed, set back to poll!\n");
951 }
952
953 ret = hns_roce_init_hem(hr_dev);
954 if (ret) {
955 dev_err(dev, "init HEM(Hardware Entry Memory) failed!\n");
956 goto error_failed_init_hem;
957 }
958
959 ret = hns_roce_setup_hca(hr_dev);
960 if (ret) {
961 dev_err(dev, "setup hca failed!\n");
962 goto error_failed_setup_hca;
963 }
964
965 if (hr_dev->hw->hw_init) {
966 ret = hr_dev->hw->hw_init(hr_dev);
967 if (ret) {
968 dev_err(dev, "hw_init failed!\n");
969 goto error_failed_engine_init;
970 }
971 }
972
973 INIT_LIST_HEAD(&hr_dev->qp_list);
974 spin_lock_init(&hr_dev->qp_list_lock);
975 INIT_LIST_HEAD(&hr_dev->dip_list);
976 spin_lock_init(&hr_dev->dip_list_lock);
977
978 ret = hns_roce_register_device(hr_dev);
979 if (ret)
980 goto error_failed_register_device;
981
982 return 0;
983
984error_failed_register_device:
985 if (hr_dev->hw->hw_exit)
986 hr_dev->hw->hw_exit(hr_dev);
987
988error_failed_engine_init:
989 hns_roce_cleanup_bitmap(hr_dev);
990
991error_failed_setup_hca:
992 hns_roce_cleanup_hem(hr_dev);
993
994error_failed_init_hem:
995 if (hr_dev->cmd_mod)
996 hns_roce_cmd_use_polling(hr_dev);
997 hr_dev->hw->cleanup_eq(hr_dev);
998
999error_failed_eq_table:
1000 hns_roce_cmd_cleanup(hr_dev);
1001
1002error_failed_cmd_init:
1003 if (hr_dev->hw->cmq_exit)
1004 hr_dev->hw->cmq_exit(hr_dev);
1005
1006error_failed_cmq_init:
1007 if (hr_dev->hw->reset) {
1008 if (hr_dev->hw->reset(hr_dev, false))
1009 dev_err(dev, "Dereset RoCE engine failed!\n");
1010 }
1011
1012 return ret;
1013}
1014
1015void hns_roce_exit(struct hns_roce_dev *hr_dev)
1016{
1017 hns_roce_unregister_device(hr_dev);
1018
1019 if (hr_dev->hw->hw_exit)
1020 hr_dev->hw->hw_exit(hr_dev);
1021 hns_roce_cleanup_bitmap(hr_dev);
1022 hns_roce_cleanup_hem(hr_dev);
1023
1024 if (hr_dev->cmd_mod)
1025 hns_roce_cmd_use_polling(hr_dev);
1026
1027 hr_dev->hw->cleanup_eq(hr_dev);
1028 hns_roce_cmd_cleanup(hr_dev);
1029 if (hr_dev->hw->cmq_exit)
1030 hr_dev->hw->cmq_exit(hr_dev);
1031 if (hr_dev->hw->reset)
1032 hr_dev->hw->reset(hr_dev, false);
1033}
1034
1035MODULE_LICENSE("Dual BSD/GPL");
1036MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
1037MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>");
1038MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
1039MODULE_DESCRIPTION("HNS RoCE Driver");
1040
