1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/acpi.h>
34#include <linux/of_platform.h>
35#include <linux/module.h>
36#include <linux/pci.h>
37#include <rdma/ib_addr.h>
38#include <rdma/ib_smi.h>
39#include <rdma/ib_user_verbs.h>
40#include <rdma/ib_cache.h>
41#include "hns_roce_common.h"
42#include "hns_roce_device.h"
43#include "hns_roce_hem.h"
44
45
46
47
48
49
50
51
52
53
54
55
56u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)
57{
58 return gid_index * hr_dev->caps.num_ports + port;
59}
60
61static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr)
62{
63 u8 phy_port;
64 u32 i;
65
66 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
67 return 0;
68
69 if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN))
70 return 0;
71
72 for (i = 0; i < ETH_ALEN; i++)
73 hr_dev->dev_addr[port][i] = addr[i];
74
75 phy_port = hr_dev->iboe.phy_port[port];
76 return hr_dev->hw->set_mac(hr_dev, phy_port, addr);
77}
78
79static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context)
80{
81 struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
82 u8 port = attr->port_num - 1;
83 int ret;
84
85 if (port >= hr_dev->caps.num_ports)
86 return -EINVAL;
87
88 ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr);
89
90 return ret;
91}
92
93static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
94{
95 struct hns_roce_dev *hr_dev = to_hr_dev(attr->device);
96 u8 port = attr->port_num - 1;
97 int ret;
98
99 if (port >= hr_dev->caps.num_ports)
100 return -EINVAL;
101
102 ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, NULL, NULL);
103
104 return ret;
105}
106
107static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
108 unsigned long event)
109{
110 struct device *dev = hr_dev->dev;
111 struct net_device *netdev;
112 int ret = 0;
113
114 netdev = hr_dev->iboe.netdevs[port];
115 if (!netdev) {
116 dev_err(dev, "Can't find netdev on port(%u)!\n", port);
117 return -ENODEV;
118 }
119
120 switch (event) {
121 case NETDEV_UP:
122 case NETDEV_CHANGE:
123 case NETDEV_REGISTER:
124 case NETDEV_CHANGEADDR:
125 ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
126 break;
127 case NETDEV_DOWN:
128
129
130
131 break;
132 default:
133 dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event));
134 break;
135 }
136
137 return ret;
138}
139
140static int hns_roce_netdev_event(struct notifier_block *self,
141 unsigned long event, void *ptr)
142{
143 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
144 struct hns_roce_ib_iboe *iboe = NULL;
145 struct hns_roce_dev *hr_dev = NULL;
146 int ret;
147 u8 port;
148
149 hr_dev = container_of(self, struct hns_roce_dev, iboe.nb);
150 iboe = &hr_dev->iboe;
151
152 for (port = 0; port < hr_dev->caps.num_ports; port++) {
153 if (dev == iboe->netdevs[port]) {
154 ret = handle_en_event(hr_dev, port, event);
155 if (ret)
156 return NOTIFY_DONE;
157 break;
158 }
159 }
160
161 return NOTIFY_DONE;
162}
163
164static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev)
165{
166 int ret;
167 u8 i;
168
169 for (i = 0; i < hr_dev->caps.num_ports; i++) {
170 if (hr_dev->hw->set_mtu)
171 hr_dev->hw->set_mtu(hr_dev, hr_dev->iboe.phy_port[i],
172 hr_dev->caps.max_mtu);
173 ret = hns_roce_set_mac(hr_dev, i,
174 hr_dev->iboe.netdevs[i]->dev_addr);
175 if (ret)
176 return ret;
177 }
178
179 return 0;
180}
181
182static int hns_roce_query_device(struct ib_device *ib_dev,
183 struct ib_device_attr *props,
184 struct ib_udata *uhw)
185{
186 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
187
188 memset(props, 0, sizeof(*props));
189
190 props->fw_ver = hr_dev->caps.fw_ver;
191 props->sys_image_guid = cpu_to_be64(hr_dev->sys_image_guid);
192 props->max_mr_size = (u64)(~(0ULL));
193 props->page_size_cap = hr_dev->caps.page_size_cap;
194 props->vendor_id = hr_dev->vendor_id;
195 props->vendor_part_id = hr_dev->vendor_part_id;
196 props->hw_ver = hr_dev->hw_rev;
197 props->max_qp = hr_dev->caps.num_qps;
198 props->max_qp_wr = hr_dev->caps.max_wqes;
199 props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT |
200 IB_DEVICE_RC_RNR_NAK_GEN;
201 props->max_send_sge = hr_dev->caps.max_sq_sg;
202 props->max_recv_sge = hr_dev->caps.max_rq_sg;
203 props->max_sge_rd = 1;
204 props->max_cq = hr_dev->caps.num_cqs;
205 props->max_cqe = hr_dev->caps.max_cqes;
206 props->max_mr = hr_dev->caps.num_mtpts;
207 props->max_pd = hr_dev->caps.num_pds;
208 props->max_qp_rd_atom = hr_dev->caps.max_qp_dest_rdma;
209 props->max_qp_init_rd_atom = hr_dev->caps.max_qp_init_rdma;
210 props->atomic_cap = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_ATOMIC ?
211 IB_ATOMIC_HCA : IB_ATOMIC_NONE;
212 props->max_pkeys = 1;
213 props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay;
214 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
215 props->max_srq = hr_dev->caps.num_srqs;
216 props->max_srq_wr = hr_dev->caps.max_srq_wrs;
217 props->max_srq_sge = hr_dev->caps.max_srq_sges;
218 }
219
220 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR) {
221 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
222 props->max_fast_reg_page_list_len = HNS_ROCE_FRMR_MAX_PA;
223 }
224
225 return 0;
226}
227
228static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
229 struct ib_port_attr *props)
230{
231 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
232 struct device *dev = hr_dev->dev;
233 struct net_device *net_dev;
234 unsigned long flags;
235 enum ib_mtu mtu;
236 u8 port;
237
238 port = port_num - 1;
239
240
241
242 props->max_mtu = hr_dev->caps.max_mtu;
243 props->gid_tbl_len = hr_dev->caps.gid_table_len[port];
244 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
245 IB_PORT_VENDOR_CLASS_SUP |
246 IB_PORT_BOOT_MGMT_SUP;
247 props->max_msg_sz = HNS_ROCE_MAX_MSG_LEN;
248 props->pkey_tbl_len = 1;
249 props->active_width = IB_WIDTH_4X;
250 props->active_speed = 1;
251
252 spin_lock_irqsave(&hr_dev->iboe.lock, flags);
253
254 net_dev = hr_dev->iboe.netdevs[port];
255 if (!net_dev) {
256 spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
257 dev_err(dev, "Find netdev %u failed!\n", port);
258 return -EINVAL;
259 }
260
261 mtu = iboe_get_mtu(net_dev->mtu);
262 props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256;
263 props->state = netif_running(net_dev) && netif_carrier_ok(net_dev) ?
264 IB_PORT_ACTIVE :
265 IB_PORT_DOWN;
266 props->phys_state = props->state == IB_PORT_ACTIVE ?
267 IB_PORT_PHYS_STATE_LINK_UP :
268 IB_PORT_PHYS_STATE_DISABLED;
269
270 spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
271
272 return 0;
273}
274
275static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device,
276 u8 port_num)
277{
278 return IB_LINK_LAYER_ETHERNET;
279}
280
281static int hns_roce_query_pkey(struct ib_device *ib_dev, u8 port, u16 index,
282 u16 *pkey)
283{
284 *pkey = PKEY_ID;
285
286 return 0;
287}
288
289static int hns_roce_modify_device(struct ib_device *ib_dev, int mask,
290 struct ib_device_modify *props)
291{
292 unsigned long flags;
293
294 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
295 return -EOPNOTSUPP;
296
297 if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
298 spin_lock_irqsave(&to_hr_dev(ib_dev)->sm_lock, flags);
299 memcpy(ib_dev->node_desc, props->node_desc, NODE_DESC_SIZE);
300 spin_unlock_irqrestore(&to_hr_dev(ib_dev)->sm_lock, flags);
301 }
302
303 return 0;
304}
305
306static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
307 struct ib_udata *udata)
308{
309 int ret;
310 struct hns_roce_ucontext *context = to_hr_ucontext(uctx);
311 struct hns_roce_ib_alloc_ucontext_resp resp = {};
312 struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
313
314 if (!hr_dev->active)
315 return -EAGAIN;
316
317 resp.qp_tab_size = hr_dev->caps.num_qps;
318
319 ret = hns_roce_uar_alloc(hr_dev, &context->uar);
320 if (ret)
321 goto error_fail_uar_alloc;
322
323 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) {
324 INIT_LIST_HEAD(&context->page_list);
325 mutex_init(&context->page_mutex);
326 }
327
328 resp.cqe_size = hr_dev->caps.cqe_sz;
329
330 ret = ib_copy_to_udata(udata, &resp,
331 min(udata->outlen, sizeof(resp)));
332 if (ret)
333 goto error_fail_copy_to_udata;
334
335 return 0;
336
337error_fail_copy_to_udata:
338 hns_roce_uar_free(hr_dev, &context->uar);
339
340error_fail_uar_alloc:
341 return ret;
342}
343
344static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext)
345{
346 struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext);
347
348 hns_roce_uar_free(to_hr_dev(ibcontext->device), &context->uar);
349}
350
351static int hns_roce_mmap(struct ib_ucontext *context,
352 struct vm_area_struct *vma)
353{
354 struct hns_roce_dev *hr_dev = to_hr_dev(context->device);
355
356 switch (vma->vm_pgoff) {
357 case 0:
358 return rdma_user_mmap_io(context, vma,
359 to_hr_ucontext(context)->uar.pfn,
360 PAGE_SIZE,
361 pgprot_noncached(vma->vm_page_prot),
362 NULL);
363
364
365 case 1:
366 if (!hr_dev->tptr_dma_addr || !hr_dev->tptr_size)
367 return -EINVAL;
368
369
370
371
372 return rdma_user_mmap_io(context, vma,
373 hr_dev->tptr_dma_addr >> PAGE_SHIFT,
374 hr_dev->tptr_size,
375 vma->vm_page_prot,
376 NULL);
377
378 default:
379 return -EINVAL;
380 }
381}
382
383static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num,
384 struct ib_port_immutable *immutable)
385{
386 struct ib_port_attr attr;
387 int ret;
388
389 ret = ib_query_port(ib_dev, port_num, &attr);
390 if (ret)
391 return ret;
392
393 immutable->pkey_tbl_len = attr.pkey_tbl_len;
394 immutable->gid_tbl_len = attr.gid_tbl_len;
395
396 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
397 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
398 if (to_hr_dev(ib_dev)->caps.flags & HNS_ROCE_CAP_FLAG_ROCE_V1_V2)
399 immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
400
401 return 0;
402}
403
404static void hns_roce_disassociate_ucontext(struct ib_ucontext *ibcontext)
405{
406}
407
408static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)
409{
410 struct hns_roce_ib_iboe *iboe = &hr_dev->iboe;
411
412 hr_dev->active = false;
413 unregister_netdevice_notifier(&iboe->nb);
414 ib_unregister_device(&hr_dev->ib_dev);
415}
416
417static const struct ib_device_ops hns_roce_dev_ops = {
418 .owner = THIS_MODULE,
419 .driver_id = RDMA_DRIVER_HNS,
420 .uverbs_abi_ver = 1,
421 .uverbs_no_driver_id_binding = 1,
422
423 .add_gid = hns_roce_add_gid,
424 .alloc_pd = hns_roce_alloc_pd,
425 .alloc_ucontext = hns_roce_alloc_ucontext,
426 .create_ah = hns_roce_create_ah,
427 .create_user_ah = hns_roce_create_ah,
428 .create_cq = hns_roce_create_cq,
429 .create_qp = hns_roce_create_qp,
430 .dealloc_pd = hns_roce_dealloc_pd,
431 .dealloc_ucontext = hns_roce_dealloc_ucontext,
432 .del_gid = hns_roce_del_gid,
433 .dereg_mr = hns_roce_dereg_mr,
434 .destroy_ah = hns_roce_destroy_ah,
435 .destroy_cq = hns_roce_destroy_cq,
436 .disassociate_ucontext = hns_roce_disassociate_ucontext,
437 .fill_res_cq_entry = hns_roce_fill_res_cq_entry,
438 .get_dma_mr = hns_roce_get_dma_mr,
439 .get_link_layer = hns_roce_get_link_layer,
440 .get_port_immutable = hns_roce_port_immutable,
441 .mmap = hns_roce_mmap,
442 .modify_device = hns_roce_modify_device,
443 .modify_qp = hns_roce_modify_qp,
444 .query_ah = hns_roce_query_ah,
445 .query_device = hns_roce_query_device,
446 .query_pkey = hns_roce_query_pkey,
447 .query_port = hns_roce_query_port,
448 .reg_user_mr = hns_roce_reg_user_mr,
449
450 INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah),
451 INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq),
452 INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd),
453 INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext),
454};
455
456static const struct ib_device_ops hns_roce_dev_mr_ops = {
457 .rereg_user_mr = hns_roce_rereg_user_mr,
458};
459
460static const struct ib_device_ops hns_roce_dev_mw_ops = {
461 .alloc_mw = hns_roce_alloc_mw,
462 .dealloc_mw = hns_roce_dealloc_mw,
463
464 INIT_RDMA_OBJ_SIZE(ib_mw, hns_roce_mw, ibmw),
465};
466
467static const struct ib_device_ops hns_roce_dev_frmr_ops = {
468 .alloc_mr = hns_roce_alloc_mr,
469 .map_mr_sg = hns_roce_map_mr_sg,
470};
471
472static const struct ib_device_ops hns_roce_dev_srq_ops = {
473 .create_srq = hns_roce_create_srq,
474 .destroy_srq = hns_roce_destroy_srq,
475
476 INIT_RDMA_OBJ_SIZE(ib_srq, hns_roce_srq, ibsrq),
477};
478
479static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
480{
481 int ret;
482 struct hns_roce_ib_iboe *iboe = NULL;
483 struct ib_device *ib_dev = NULL;
484 struct device *dev = hr_dev->dev;
485 unsigned int i;
486
487 iboe = &hr_dev->iboe;
488 spin_lock_init(&iboe->lock);
489
490 ib_dev = &hr_dev->ib_dev;
491
492 ib_dev->node_type = RDMA_NODE_IB_CA;
493 ib_dev->dev.parent = dev;
494
495 ib_dev->phys_port_cnt = hr_dev->caps.num_ports;
496 ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey;
497 ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors;
498
499 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR)
500 ib_set_device_ops(ib_dev, &hns_roce_dev_mr_ops);
501
502
503 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW)
504 ib_set_device_ops(ib_dev, &hns_roce_dev_mw_ops);
505
506
507 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR)
508 ib_set_device_ops(ib_dev, &hns_roce_dev_frmr_ops);
509
510
511 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
512 ib_set_device_ops(ib_dev, &hns_roce_dev_srq_ops);
513 ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops);
514 }
515
516 ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops);
517 ib_set_device_ops(ib_dev, &hns_roce_dev_ops);
518 for (i = 0; i < hr_dev->caps.num_ports; i++) {
519 if (!hr_dev->iboe.netdevs[i])
520 continue;
521
522 ret = ib_device_set_netdev(ib_dev, hr_dev->iboe.netdevs[i],
523 i + 1);
524 if (ret)
525 return ret;
526 }
527 dma_set_max_seg_size(dev, UINT_MAX);
528 ret = ib_register_device(ib_dev, "hns_%d", dev);
529 if (ret) {
530 dev_err(dev, "ib_register_device failed!\n");
531 return ret;
532 }
533
534 ret = hns_roce_setup_mtu_mac(hr_dev);
535 if (ret) {
536 dev_err(dev, "setup_mtu_mac failed!\n");
537 goto error_failed_setup_mtu_mac;
538 }
539
540 iboe->nb.notifier_call = hns_roce_netdev_event;
541 ret = register_netdevice_notifier(&iboe->nb);
542 if (ret) {
543 dev_err(dev, "register_netdevice_notifier failed!\n");
544 goto error_failed_setup_mtu_mac;
545 }
546
547 hr_dev->active = true;
548 return 0;
549
550error_failed_setup_mtu_mac:
551 ib_unregister_device(ib_dev);
552
553 return ret;
554}
555
556static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
557{
558 struct device *dev = hr_dev->dev;
559 int ret;
560
561 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table,
562 HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz,
563 hr_dev->caps.num_mtpts, 1);
564 if (ret) {
565 dev_err(dev, "Failed to init MTPT context memory, aborting.\n");
566 return ret;
567 }
568
569 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table,
570 HEM_TYPE_QPC, hr_dev->caps.qpc_sz,
571 hr_dev->caps.num_qps, 1);
572 if (ret) {
573 dev_err(dev, "Failed to init QP context memory, aborting.\n");
574 goto err_unmap_dmpt;
575 }
576
577 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.irrl_table,
578 HEM_TYPE_IRRL,
579 hr_dev->caps.irrl_entry_sz *
580 hr_dev->caps.max_qp_init_rdma,
581 hr_dev->caps.num_qps, 1);
582 if (ret) {
583 dev_err(dev, "Failed to init irrl_table memory, aborting.\n");
584 goto err_unmap_qp;
585 }
586
587 if (hr_dev->caps.trrl_entry_sz) {
588 ret = hns_roce_init_hem_table(hr_dev,
589 &hr_dev->qp_table.trrl_table,
590 HEM_TYPE_TRRL,
591 hr_dev->caps.trrl_entry_sz *
592 hr_dev->caps.max_qp_dest_rdma,
593 hr_dev->caps.num_qps, 1);
594 if (ret) {
595 dev_err(dev,
596 "Failed to init trrl_table memory, aborting.\n");
597 goto err_unmap_irrl;
598 }
599 }
600
601 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table,
602 HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz,
603 hr_dev->caps.num_cqs, 1);
604 if (ret) {
605 dev_err(dev, "Failed to init CQ context memory, aborting.\n");
606 goto err_unmap_trrl;
607 }
608
609 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
610 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->srq_table.table,
611 HEM_TYPE_SRQC,
612 hr_dev->caps.srqc_entry_sz,
613 hr_dev->caps.num_srqs, 1);
614 if (ret) {
615 dev_err(dev,
616 "Failed to init SRQ context memory, aborting.\n");
617 goto err_unmap_cq;
618 }
619 }
620
621 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
622 ret = hns_roce_init_hem_table(hr_dev,
623 &hr_dev->qp_table.sccc_table,
624 HEM_TYPE_SCCC,
625 hr_dev->caps.sccc_sz,
626 hr_dev->caps.num_qps, 1);
627 if (ret) {
628 dev_err(dev,
629 "Failed to init SCC context memory, aborting.\n");
630 goto err_unmap_srq;
631 }
632 }
633
634 if (hr_dev->caps.qpc_timer_entry_sz) {
635 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qpc_timer_table,
636 HEM_TYPE_QPC_TIMER,
637 hr_dev->caps.qpc_timer_entry_sz,
638 hr_dev->caps.num_qpc_timer, 1);
639 if (ret) {
640 dev_err(dev,
641 "Failed to init QPC timer memory, aborting.\n");
642 goto err_unmap_ctx;
643 }
644 }
645
646 if (hr_dev->caps.cqc_timer_entry_sz) {
647 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cqc_timer_table,
648 HEM_TYPE_CQC_TIMER,
649 hr_dev->caps.cqc_timer_entry_sz,
650 hr_dev->caps.num_cqc_timer, 1);
651 if (ret) {
652 dev_err(dev,
653 "Failed to init CQC timer memory, aborting.\n");
654 goto err_unmap_qpc_timer;
655 }
656 }
657
658 if (hr_dev->caps.gmv_entry_sz) {
659 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->gmv_table,
660 HEM_TYPE_GMV,
661 hr_dev->caps.gmv_entry_sz,
662 hr_dev->caps.gmv_entry_num, 1);
663 if (ret) {
664 dev_err(dev,
665 "failed to init gmv table memory, ret = %d\n",
666 ret);
667 goto err_unmap_cqc_timer;
668 }
669 }
670
671 return 0;
672
673err_unmap_cqc_timer:
674 if (hr_dev->caps.cqc_timer_entry_sz)
675 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cqc_timer_table);
676
677err_unmap_qpc_timer:
678 if (hr_dev->caps.qpc_timer_entry_sz)
679 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qpc_timer_table);
680
681err_unmap_ctx:
682 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
683 hns_roce_cleanup_hem_table(hr_dev,
684 &hr_dev->qp_table.sccc_table);
685err_unmap_srq:
686 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
687 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table);
688
689err_unmap_cq:
690 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
691
692err_unmap_trrl:
693 if (hr_dev->caps.trrl_entry_sz)
694 hns_roce_cleanup_hem_table(hr_dev,
695 &hr_dev->qp_table.trrl_table);
696
697err_unmap_irrl:
698 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
699
700err_unmap_qp:
701 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
702
703err_unmap_dmpt:
704 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
705
706 return ret;
707}
708
709
710
711
712
713
714static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)
715{
716 struct device *dev = hr_dev->dev;
717 int ret;
718
719 spin_lock_init(&hr_dev->sm_lock);
720 spin_lock_init(&hr_dev->bt_cmd_lock);
721
722 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) {
723 INIT_LIST_HEAD(&hr_dev->pgdir_list);
724 mutex_init(&hr_dev->pgdir_mutex);
725 }
726
727 ret = hns_roce_init_uar_table(hr_dev);
728 if (ret) {
729 dev_err(dev, "Failed to initialize uar table. aborting\n");
730 return ret;
731 }
732
733 ret = hns_roce_uar_alloc(hr_dev, &hr_dev->priv_uar);
734 if (ret) {
735 dev_err(dev, "Failed to allocate priv_uar.\n");
736 goto err_uar_table_free;
737 }
738
739 ret = hns_roce_init_pd_table(hr_dev);
740 if (ret) {
741 dev_err(dev, "Failed to init protected domain table.\n");
742 goto err_uar_alloc_free;
743 }
744
745 ret = hns_roce_init_mr_table(hr_dev);
746 if (ret) {
747 dev_err(dev, "Failed to init memory region table.\n");
748 goto err_pd_table_free;
749 }
750
751 ret = hns_roce_init_cq_table(hr_dev);
752 if (ret) {
753 dev_err(dev, "Failed to init completion queue table.\n");
754 goto err_mr_table_free;
755 }
756
757 ret = hns_roce_init_qp_table(hr_dev);
758 if (ret) {
759 dev_err(dev, "Failed to init queue pair table.\n");
760 goto err_cq_table_free;
761 }
762
763 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) {
764 ret = hns_roce_init_srq_table(hr_dev);
765 if (ret) {
766 dev_err(dev,
767 "Failed to init share receive queue table.\n");
768 goto err_qp_table_free;
769 }
770 }
771
772 return 0;
773
774err_qp_table_free:
775 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
776 hns_roce_cleanup_qp_table(hr_dev);
777
778err_cq_table_free:
779 hns_roce_cleanup_cq_table(hr_dev);
780
781err_mr_table_free:
782 hns_roce_cleanup_mr_table(hr_dev);
783
784err_pd_table_free:
785 hns_roce_cleanup_pd_table(hr_dev);
786
787err_uar_alloc_free:
788 hns_roce_uar_free(hr_dev, &hr_dev->priv_uar);
789
790err_uar_table_free:
791 hns_roce_cleanup_uar_table(hr_dev);
792 return ret;
793}
794
795static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq)
796{
797 struct hns_roce_cq *hr_cq = to_hr_cq(cq);
798 unsigned long flags;
799
800 spin_lock_irqsave(&hr_cq->lock, flags);
801 if (cq->comp_handler) {
802 if (!hr_cq->is_armed) {
803 hr_cq->is_armed = 1;
804 list_add_tail(&hr_cq->node, cq_list);
805 }
806 }
807 spin_unlock_irqrestore(&hr_cq->lock, flags);
808}
809
810void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev)
811{
812 struct hns_roce_qp *hr_qp;
813 struct hns_roce_cq *hr_cq;
814 struct list_head cq_list;
815 unsigned long flags_qp;
816 unsigned long flags;
817
818 INIT_LIST_HEAD(&cq_list);
819
820 spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
821 list_for_each_entry(hr_qp, &hr_dev->qp_list, node) {
822 spin_lock_irqsave(&hr_qp->sq.lock, flags_qp);
823 if (hr_qp->sq.tail != hr_qp->sq.head)
824 check_and_get_armed_cq(&cq_list, hr_qp->ibqp.send_cq);
825 spin_unlock_irqrestore(&hr_qp->sq.lock, flags_qp);
826
827 spin_lock_irqsave(&hr_qp->rq.lock, flags_qp);
828 if ((!hr_qp->ibqp.srq) && (hr_qp->rq.tail != hr_qp->rq.head))
829 check_and_get_armed_cq(&cq_list, hr_qp->ibqp.recv_cq);
830 spin_unlock_irqrestore(&hr_qp->rq.lock, flags_qp);
831 }
832
833 list_for_each_entry(hr_cq, &cq_list, node)
834 hns_roce_cq_completion(hr_dev, hr_cq->cqn);
835
836 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
837}
838
839int hns_roce_init(struct hns_roce_dev *hr_dev)
840{
841 struct device *dev = hr_dev->dev;
842 int ret;
843
844 if (hr_dev->hw->reset) {
845 ret = hr_dev->hw->reset(hr_dev, true);
846 if (ret) {
847 dev_err(dev, "Reset RoCE engine failed!\n");
848 return ret;
849 }
850 }
851 hr_dev->is_reset = false;
852
853 if (hr_dev->hw->cmq_init) {
854 ret = hr_dev->hw->cmq_init(hr_dev);
855 if (ret) {
856 dev_err(dev, "Init RoCE Command Queue failed!\n");
857 goto error_failed_cmq_init;
858 }
859 }
860
861 ret = hr_dev->hw->hw_profile(hr_dev);
862 if (ret) {
863 dev_err(dev, "Get RoCE engine profile failed!\n");
864 goto error_failed_cmd_init;
865 }
866
867 ret = hns_roce_cmd_init(hr_dev);
868 if (ret) {
869 dev_err(dev, "cmd init failed!\n");
870 goto error_failed_cmd_init;
871 }
872
873
874 ret = hr_dev->hw->init_eq(hr_dev);
875 if (ret) {
876 dev_err(dev, "eq init failed!\n");
877 goto error_failed_eq_table;
878 }
879
880 if (hr_dev->cmd_mod) {
881 ret = hns_roce_cmd_use_events(hr_dev);
882 if (ret) {
883 dev_warn(dev,
884 "Cmd event mode failed, set back to poll!\n");
885 hns_roce_cmd_use_polling(hr_dev);
886 }
887 }
888
889 ret = hns_roce_init_hem(hr_dev);
890 if (ret) {
891 dev_err(dev, "init HEM(Hardware Entry Memory) failed!\n");
892 goto error_failed_init_hem;
893 }
894
895 ret = hns_roce_setup_hca(hr_dev);
896 if (ret) {
897 dev_err(dev, "setup hca failed!\n");
898 goto error_failed_setup_hca;
899 }
900
901 if (hr_dev->hw->hw_init) {
902 ret = hr_dev->hw->hw_init(hr_dev);
903 if (ret) {
904 dev_err(dev, "hw_init failed!\n");
905 goto error_failed_engine_init;
906 }
907 }
908
909 INIT_LIST_HEAD(&hr_dev->qp_list);
910 spin_lock_init(&hr_dev->qp_list_lock);
911
912 ret = hns_roce_register_device(hr_dev);
913 if (ret)
914 goto error_failed_register_device;
915
916 return 0;
917
918error_failed_register_device:
919 if (hr_dev->hw->hw_exit)
920 hr_dev->hw->hw_exit(hr_dev);
921
922error_failed_engine_init:
923 hns_roce_cleanup_bitmap(hr_dev);
924
925error_failed_setup_hca:
926 hns_roce_cleanup_hem(hr_dev);
927
928error_failed_init_hem:
929 if (hr_dev->cmd_mod)
930 hns_roce_cmd_use_polling(hr_dev);
931 hr_dev->hw->cleanup_eq(hr_dev);
932
933error_failed_eq_table:
934 hns_roce_cmd_cleanup(hr_dev);
935
936error_failed_cmd_init:
937 if (hr_dev->hw->cmq_exit)
938 hr_dev->hw->cmq_exit(hr_dev);
939
940error_failed_cmq_init:
941 if (hr_dev->hw->reset) {
942 if (hr_dev->hw->reset(hr_dev, false))
943 dev_err(dev, "Dereset RoCE engine failed!\n");
944 }
945
946 return ret;
947}
948
949void hns_roce_exit(struct hns_roce_dev *hr_dev)
950{
951 hns_roce_unregister_device(hr_dev);
952
953 if (hr_dev->hw->hw_exit)
954 hr_dev->hw->hw_exit(hr_dev);
955 hns_roce_cleanup_bitmap(hr_dev);
956 hns_roce_cleanup_hem(hr_dev);
957
958 if (hr_dev->cmd_mod)
959 hns_roce_cmd_use_polling(hr_dev);
960
961 hr_dev->hw->cleanup_eq(hr_dev);
962 hns_roce_cmd_cleanup(hr_dev);
963 if (hr_dev->hw->cmq_exit)
964 hr_dev->hw->cmq_exit(hr_dev);
965 if (hr_dev->hw->reset)
966 hr_dev->hw->reset(hr_dev, false);
967}
968
969MODULE_LICENSE("Dual BSD/GPL");
970MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
971MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>");
972MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
973MODULE_DESCRIPTION("HNS RoCE Driver");
974
