1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/platform_device.h>
35#include <linux/vmalloc.h>
36#include <rdma/ib_umem.h>
37#include "hns_roce_device.h"
38#include "hns_roce_cmd.h"
39#include "hns_roce_hem.h"
40
41static u32 hw_index_to_key(unsigned long ind)
42{
43 return (u32)(ind >> 24) | (ind << 8);
44}
45
46unsigned long key_to_hw_index(u32 key)
47{
48 return (key << 24) | (key >> 8);
49}
50EXPORT_SYMBOL_GPL(key_to_hw_index);
51
52static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,
53 struct hns_roce_cmd_mailbox *mailbox,
54 unsigned long mpt_index)
55{
56 return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0,
57 HNS_ROCE_CMD_SW2HW_MPT,
58 HNS_ROCE_CMD_TIMEOUT_MSECS);
59}
60
61int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
62 struct hns_roce_cmd_mailbox *mailbox,
63 unsigned long mpt_index)
64{
65 return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0,
66 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,
67 HNS_ROCE_CMD_TIMEOUT_MSECS);
68}
69EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt);
70
71static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
72 unsigned long *seg)
73{
74 int o;
75 u32 m;
76
77 spin_lock(&buddy->lock);
78
79 for (o = order; o <= buddy->max_order; ++o) {
80 if (buddy->num_free[o]) {
81 m = 1 << (buddy->max_order - o);
82 *seg = find_first_bit(buddy->bits[o], m);
83 if (*seg < m)
84 goto found;
85 }
86 }
87 spin_unlock(&buddy->lock);
88 return -1;
89
90 found:
91 clear_bit(*seg, buddy->bits[o]);
92 --buddy->num_free[o];
93
94 while (o > order) {
95 --o;
96 *seg <<= 1;
97 set_bit(*seg ^ 1, buddy->bits[o]);
98 ++buddy->num_free[o];
99 }
100
101 spin_unlock(&buddy->lock);
102
103 *seg <<= order;
104 return 0;
105}
106
107static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg,
108 int order)
109{
110 seg >>= order;
111
112 spin_lock(&buddy->lock);
113
114 while (test_bit(seg ^ 1, buddy->bits[order])) {
115 clear_bit(seg ^ 1, buddy->bits[order]);
116 --buddy->num_free[order];
117 seg >>= 1;
118 ++order;
119 }
120
121 set_bit(seg, buddy->bits[order]);
122 ++buddy->num_free[order];
123
124 spin_unlock(&buddy->lock);
125}
126
127static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
128{
129 int i, s;
130
131 buddy->max_order = max_order;
132 spin_lock_init(&buddy->lock);
133 buddy->bits = kcalloc(buddy->max_order + 1,
134 sizeof(*buddy->bits),
135 GFP_KERNEL);
136 buddy->num_free = kcalloc(buddy->max_order + 1,
137 sizeof(*buddy->num_free),
138 GFP_KERNEL);
139 if (!buddy->bits || !buddy->num_free)
140 goto err_out;
141
142 for (i = 0; i <= buddy->max_order; ++i) {
143 s = BITS_TO_LONGS(1 << (buddy->max_order - i));
144 buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL |
145 __GFP_NOWARN);
146 if (!buddy->bits[i]) {
147 buddy->bits[i] = vzalloc(array_size(s, sizeof(long)));
148 if (!buddy->bits[i])
149 goto err_out_free;
150 }
151 }
152
153 set_bit(0, buddy->bits[buddy->max_order]);
154 buddy->num_free[buddy->max_order] = 1;
155
156 return 0;
157
158err_out_free:
159 for (i = 0; i <= buddy->max_order; ++i)
160 kvfree(buddy->bits[i]);
161
162err_out:
163 kfree(buddy->bits);
164 kfree(buddy->num_free);
165 return -ENOMEM;
166}
167
168static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)
169{
170 int i;
171
172 for (i = 0; i <= buddy->max_order; ++i)
173 kvfree(buddy->bits[i]);
174
175 kfree(buddy->bits);
176 kfree(buddy->num_free);
177}
178
179static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order,
180 unsigned long *seg, u32 mtt_type)
181{
182 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
183 struct hns_roce_hem_table *table;
184 struct hns_roce_buddy *buddy;
185 int ret;
186
187 switch (mtt_type) {
188 case MTT_TYPE_WQE:
189 buddy = &mr_table->mtt_buddy;
190 table = &mr_table->mtt_table;
191 break;
192 case MTT_TYPE_CQE:
193 buddy = &mr_table->mtt_cqe_buddy;
194 table = &mr_table->mtt_cqe_table;
195 break;
196 case MTT_TYPE_SRQWQE:
197 buddy = &mr_table->mtt_srqwqe_buddy;
198 table = &mr_table->mtt_srqwqe_table;
199 break;
200 case MTT_TYPE_IDX:
201 buddy = &mr_table->mtt_idx_buddy;
202 table = &mr_table->mtt_idx_table;
203 break;
204 default:
205 dev_err(hr_dev->dev, "Unsupport MTT table type: %d\n",
206 mtt_type);
207 return -EINVAL;
208 }
209
210 ret = hns_roce_buddy_alloc(buddy, order, seg);
211 if (ret == -1)
212 return -1;
213
214 if (hns_roce_table_get_range(hr_dev, table, *seg,
215 *seg + (1 << order) - 1)) {
216 hns_roce_buddy_free(buddy, *seg, order);
217 return -1;
218 }
219
220 return 0;
221}
222
223int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
224 struct hns_roce_mtt *mtt)
225{
226 int ret;
227 int i;
228
229
230 if (!npages) {
231 mtt->order = -1;
232 mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT;
233 return 0;
234 }
235
236
237 mtt->page_shift = page_shift;
238
239
240 for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages;
241 i <<= 1)
242 ++mtt->order;
243
244
245 ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg,
246 mtt->mtt_type);
247 if (ret == -1)
248 return -ENOMEM;
249
250 return 0;
251}
252
253void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)
254{
255 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
256
257 if (mtt->order < 0)
258 return;
259
260 switch (mtt->mtt_type) {
261 case MTT_TYPE_WQE:
262 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,
263 mtt->order);
264 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,
265 mtt->first_seg,
266 mtt->first_seg + (1 << mtt->order) - 1);
267 break;
268 case MTT_TYPE_CQE:
269 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,
270 mtt->order);
271 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,
272 mtt->first_seg,
273 mtt->first_seg + (1 << mtt->order) - 1);
274 break;
275 case MTT_TYPE_SRQWQE:
276 hns_roce_buddy_free(&mr_table->mtt_srqwqe_buddy, mtt->first_seg,
277 mtt->order);
278 hns_roce_table_put_range(hr_dev, &mr_table->mtt_srqwqe_table,
279 mtt->first_seg,
280 mtt->first_seg + (1 << mtt->order) - 1);
281 break;
282 case MTT_TYPE_IDX:
283 hns_roce_buddy_free(&mr_table->mtt_idx_buddy, mtt->first_seg,
284 mtt->order);
285 hns_roce_table_put_range(hr_dev, &mr_table->mtt_idx_table,
286 mtt->first_seg,
287 mtt->first_seg + (1 << mtt->order) - 1);
288 break;
289 default:
290 dev_err(hr_dev->dev,
291 "Unsupport mtt type %d, clean mtt failed\n",
292 mtt->mtt_type);
293 break;
294 }
295}
296EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);
297
298static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
299 struct hns_roce_mr *mr, int err_loop_index,
300 int loop_i, int loop_j)
301{
302 struct device *dev = hr_dev->dev;
303 u32 mhop_num;
304 u32 pbl_bt_sz;
305 u64 bt_idx;
306 int i, j;
307
308 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
309 mhop_num = hr_dev->caps.pbl_hop_num;
310
311 i = loop_i;
312 if (mhop_num == 3 && err_loop_index == 2) {
313 for (; i >= 0; i--) {
314 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
315 mr->pbl_l1_dma_addr[i]);
316
317 for (j = 0; j < pbl_bt_sz / 8; j++) {
318 if (i == loop_i && j >= loop_j)
319 break;
320
321 bt_idx = i * pbl_bt_sz / 8 + j;
322 dma_free_coherent(dev, pbl_bt_sz,
323 mr->pbl_bt_l2[bt_idx],
324 mr->pbl_l2_dma_addr[bt_idx]);
325 }
326 }
327 } else if (mhop_num == 3 && err_loop_index == 1) {
328 for (i -= 1; i >= 0; i--) {
329 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
330 mr->pbl_l1_dma_addr[i]);
331
332 for (j = 0; j < pbl_bt_sz / 8; j++) {
333 bt_idx = i * pbl_bt_sz / 8 + j;
334 dma_free_coherent(dev, pbl_bt_sz,
335 mr->pbl_bt_l2[bt_idx],
336 mr->pbl_l2_dma_addr[bt_idx]);
337 }
338 }
339 } else if (mhop_num == 2 && err_loop_index == 1) {
340 for (i -= 1; i >= 0; i--)
341 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
342 mr->pbl_l1_dma_addr[i]);
343 } else {
344 dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.",
345 mhop_num, err_loop_index);
346 return;
347 }
348
349 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr);
350 mr->pbl_bt_l0 = NULL;
351 mr->pbl_l0_dma_addr = 0;
352}
353
354
355static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages,
356 struct hns_roce_mr *mr)
357{
358 struct device *dev = hr_dev->dev;
359 int mr_alloc_done = 0;
360 int npages_allocated;
361 int i = 0, j = 0;
362 u32 pbl_bt_sz;
363 u32 mhop_num;
364 u64 pbl_last_bt_num;
365 u64 pbl_bt_cnt = 0;
366 u64 bt_idx;
367 u64 size;
368
369 mhop_num = (mr->type == MR_TYPE_FRMR ? 1 : hr_dev->caps.pbl_hop_num);
370 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
371 pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);
372
373 if (mhop_num == HNS_ROCE_HOP_NUM_0)
374 return 0;
375
376
377 if (mhop_num == 1) {
378 if (npages > pbl_bt_sz / 8) {
379 dev_err(dev, "npages %d is larger than buf_pg_sz!",
380 npages);
381 return -EINVAL;
382 }
383 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
384 &(mr->pbl_dma_addr),
385 GFP_KERNEL);
386 if (!mr->pbl_buf)
387 return -ENOMEM;
388
389 mr->pbl_size = npages;
390 mr->pbl_ba = mr->pbl_dma_addr;
391 mr->pbl_hop_num = mhop_num;
392 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
393 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
394 return 0;
395 }
396
397 mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8,
398 sizeof(*mr->pbl_l1_dma_addr),
399 GFP_KERNEL);
400 if (!mr->pbl_l1_dma_addr)
401 return -ENOMEM;
402
403 mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1),
404 GFP_KERNEL);
405 if (!mr->pbl_bt_l1)
406 goto err_kcalloc_bt_l1;
407
408 if (mhop_num == 3) {
409 mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num,
410 sizeof(*mr->pbl_l2_dma_addr),
411 GFP_KERNEL);
412 if (!mr->pbl_l2_dma_addr)
413 goto err_kcalloc_l2_dma;
414
415 mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num,
416 sizeof(*mr->pbl_bt_l2),
417 GFP_KERNEL);
418 if (!mr->pbl_bt_l2)
419 goto err_kcalloc_bt_l2;
420 }
421
422
423 mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz,
424 &(mr->pbl_l0_dma_addr),
425 GFP_KERNEL);
426 if (!mr->pbl_bt_l0)
427 goto err_dma_alloc_l0;
428
429 if (mhop_num == 2) {
430
431 for (i = 0; i < pbl_bt_sz / 8; i++) {
432 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
433 size = pbl_bt_sz;
434 } else {
435 npages_allocated = i * (pbl_bt_sz / 8);
436 size = (npages - npages_allocated) * 8;
437 }
438 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size,
439 &(mr->pbl_l1_dma_addr[i]),
440 GFP_KERNEL);
441 if (!mr->pbl_bt_l1[i]) {
442 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
443 goto err_dma_alloc_l0;
444 }
445
446 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
447
448 pbl_bt_cnt++;
449 if (pbl_bt_cnt >= pbl_last_bt_num)
450 break;
451 }
452 } else if (mhop_num == 3) {
453
454 for (i = 0; i < pbl_bt_sz / 8; i++) {
455 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz,
456 &(mr->pbl_l1_dma_addr[i]),
457 GFP_KERNEL);
458 if (!mr->pbl_bt_l1[i]) {
459 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
460 goto err_dma_alloc_l0;
461 }
462
463 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
464
465 for (j = 0; j < pbl_bt_sz / 8; j++) {
466 bt_idx = i * pbl_bt_sz / 8 + j;
467
468 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
469 size = pbl_bt_sz;
470 } else {
471 npages_allocated = bt_idx *
472 (pbl_bt_sz / 8);
473 size = (npages - npages_allocated) * 8;
474 }
475 mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent(
476 dev, size,
477 &(mr->pbl_l2_dma_addr[bt_idx]),
478 GFP_KERNEL);
479 if (!mr->pbl_bt_l2[bt_idx]) {
480 hns_roce_loop_free(hr_dev, mr, 2, i, j);
481 goto err_dma_alloc_l0;
482 }
483
484 *(mr->pbl_bt_l1[i] + j) =
485 mr->pbl_l2_dma_addr[bt_idx];
486
487 pbl_bt_cnt++;
488 if (pbl_bt_cnt >= pbl_last_bt_num) {
489 mr_alloc_done = 1;
490 break;
491 }
492 }
493
494 if (mr_alloc_done)
495 break;
496 }
497 }
498
499 mr->l0_chunk_last_num = i + 1;
500 if (mhop_num == 3)
501 mr->l1_chunk_last_num = j + 1;
502
503 mr->pbl_size = npages;
504 mr->pbl_ba = mr->pbl_l0_dma_addr;
505 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num;
506 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
507 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
508
509 return 0;
510
511err_dma_alloc_l0:
512 kfree(mr->pbl_bt_l2);
513 mr->pbl_bt_l2 = NULL;
514
515err_kcalloc_bt_l2:
516 kfree(mr->pbl_l2_dma_addr);
517 mr->pbl_l2_dma_addr = NULL;
518
519err_kcalloc_l2_dma:
520 kfree(mr->pbl_bt_l1);
521 mr->pbl_bt_l1 = NULL;
522
523err_kcalloc_bt_l1:
524 kfree(mr->pbl_l1_dma_addr);
525 mr->pbl_l1_dma_addr = NULL;
526
527 return -ENOMEM;
528}
529
530static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
531 u64 size, u32 access, int npages,
532 struct hns_roce_mr *mr)
533{
534 struct device *dev = hr_dev->dev;
535 unsigned long index = 0;
536 int ret = 0;
537
538
539 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
540 if (ret == -1)
541 return -ENOMEM;
542
543 mr->iova = iova;
544 mr->size = size;
545 mr->pd = pd;
546 mr->access = access;
547 mr->enabled = 0;
548 mr->key = hw_index_to_key(index);
549
550 if (size == ~0ull) {
551 mr->pbl_buf = NULL;
552 mr->pbl_dma_addr = 0;
553
554 mr->pbl_bt_l2 = NULL;
555 mr->pbl_bt_l1 = NULL;
556 mr->pbl_bt_l0 = NULL;
557 mr->pbl_l2_dma_addr = NULL;
558 mr->pbl_l1_dma_addr = NULL;
559 mr->pbl_l0_dma_addr = 0;
560 } else {
561 if (!hr_dev->caps.pbl_hop_num) {
562 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
563 &(mr->pbl_dma_addr),
564 GFP_KERNEL);
565 if (!mr->pbl_buf)
566 return -ENOMEM;
567 } else {
568 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
569 }
570 }
571
572 return ret;
573}
574
575static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
576 struct hns_roce_mr *mr)
577{
578 struct device *dev = hr_dev->dev;
579 int npages_allocated;
580 int npages;
581 int i, j;
582 u32 pbl_bt_sz;
583 u32 mhop_num;
584 u64 bt_idx;
585
586 npages = mr->pbl_size;
587 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
588 mhop_num = (mr->type == MR_TYPE_FRMR) ? 1 : hr_dev->caps.pbl_hop_num;
589
590 if (mhop_num == HNS_ROCE_HOP_NUM_0)
591 return;
592
593
594 if (mhop_num == 1) {
595 dma_free_coherent(dev, (unsigned int)(npages * 8),
596 mr->pbl_buf, mr->pbl_dma_addr);
597 return;
598 }
599
600 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0,
601 mr->pbl_l0_dma_addr);
602
603 if (mhop_num == 2) {
604 for (i = 0; i < mr->l0_chunk_last_num; i++) {
605 if (i == mr->l0_chunk_last_num - 1) {
606 npages_allocated = i * (pbl_bt_sz / 8);
607
608 dma_free_coherent(dev,
609 (npages - npages_allocated) * 8,
610 mr->pbl_bt_l1[i],
611 mr->pbl_l1_dma_addr[i]);
612
613 break;
614 }
615
616 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
617 mr->pbl_l1_dma_addr[i]);
618 }
619 } else if (mhop_num == 3) {
620 for (i = 0; i < mr->l0_chunk_last_num; i++) {
621 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
622 mr->pbl_l1_dma_addr[i]);
623
624 for (j = 0; j < pbl_bt_sz / 8; j++) {
625 bt_idx = i * (pbl_bt_sz / 8) + j;
626
627 if ((i == mr->l0_chunk_last_num - 1)
628 && j == mr->l1_chunk_last_num - 1) {
629 npages_allocated = bt_idx *
630 (pbl_bt_sz / 8);
631
632 dma_free_coherent(dev,
633 (npages - npages_allocated) * 8,
634 mr->pbl_bt_l2[bt_idx],
635 mr->pbl_l2_dma_addr[bt_idx]);
636
637 break;
638 }
639
640 dma_free_coherent(dev, pbl_bt_sz,
641 mr->pbl_bt_l2[bt_idx],
642 mr->pbl_l2_dma_addr[bt_idx]);
643 }
644 }
645 }
646
647 kfree(mr->pbl_bt_l1);
648 kfree(mr->pbl_l1_dma_addr);
649 mr->pbl_bt_l1 = NULL;
650 mr->pbl_l1_dma_addr = NULL;
651 if (mhop_num == 3) {
652 kfree(mr->pbl_bt_l2);
653 kfree(mr->pbl_l2_dma_addr);
654 mr->pbl_bt_l2 = NULL;
655 mr->pbl_l2_dma_addr = NULL;
656 }
657}
658
659static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
660 struct hns_roce_mr *mr)
661{
662 struct device *dev = hr_dev->dev;
663 int npages = 0;
664 int ret;
665
666 if (mr->enabled) {
667 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
668 & (hr_dev->caps.num_mtpts - 1));
669 if (ret)
670 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
671 }
672
673 if (mr->size != ~0ULL) {
674 if (mr->type == MR_TYPE_MR)
675 npages = ib_umem_page_count(mr->umem);
676
677 if (!hr_dev->caps.pbl_hop_num)
678 dma_free_coherent(dev, (unsigned int)(npages * 8),
679 mr->pbl_buf, mr->pbl_dma_addr);
680 else
681 hns_roce_mhop_free(hr_dev, mr);
682 }
683
684 if (mr->enabled)
685 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
686 key_to_hw_index(mr->key));
687
688 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
689 key_to_hw_index(mr->key), BITMAP_NO_RR);
690}
691
692static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
693 struct hns_roce_mr *mr)
694{
695 int ret;
696 unsigned long mtpt_idx = key_to_hw_index(mr->key);
697 struct device *dev = hr_dev->dev;
698 struct hns_roce_cmd_mailbox *mailbox;
699 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
700
701
702 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
703 if (ret)
704 return ret;
705
706
707 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
708 if (IS_ERR(mailbox)) {
709 ret = PTR_ERR(mailbox);
710 goto err_table;
711 }
712
713 if (mr->type != MR_TYPE_FRMR)
714 ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx);
715 else
716 ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr);
717 if (ret) {
718 dev_err(dev, "Write mtpt fail!\n");
719 goto err_page;
720 }
721
722 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,
723 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
724 if (ret) {
725 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
726 goto err_page;
727 }
728
729 mr->enabled = 1;
730 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
731
732 return 0;
733
734err_page:
735 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
736
737err_table:
738 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
739 return ret;
740}
741
742static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev,
743 struct hns_roce_mtt *mtt, u32 start_index,
744 u32 npages, u64 *page_list)
745{
746 struct hns_roce_hem_table *table;
747 dma_addr_t dma_handle;
748 __le64 *mtts;
749 u32 s = start_index * sizeof(u64);
750 u32 bt_page_size;
751 u32 i;
752
753 switch (mtt->mtt_type) {
754 case MTT_TYPE_WQE:
755 table = &hr_dev->mr_table.mtt_table;
756 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
757 break;
758 case MTT_TYPE_CQE:
759 table = &hr_dev->mr_table.mtt_cqe_table;
760 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
761 break;
762 case MTT_TYPE_SRQWQE:
763 table = &hr_dev->mr_table.mtt_srqwqe_table;
764 bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
765 break;
766 case MTT_TYPE_IDX:
767 table = &hr_dev->mr_table.mtt_idx_table;
768 bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
769 break;
770 default:
771 return -EINVAL;
772 }
773
774
775 if (start_index / (bt_page_size / sizeof(u64)) !=
776 (start_index + npages - 1) / (bt_page_size / sizeof(u64)))
777 return -EINVAL;
778
779 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
780 return -EINVAL;
781
782 mtts = hns_roce_table_find(hr_dev, table,
783 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz,
784 &dma_handle);
785 if (!mtts)
786 return -ENOMEM;
787
788
789 for (i = 0; i < npages; ++i) {
790 if (!hr_dev->caps.mtt_hop_num)
791 mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT);
792 else
793 mtts[i] = cpu_to_le64(page_list[i]);
794 }
795
796 return 0;
797}
798
799static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev,
800 struct hns_roce_mtt *mtt, u32 start_index,
801 u32 npages, u64 *page_list)
802{
803 int chunk;
804 int ret;
805 u32 bt_page_size;
806
807 if (mtt->order < 0)
808 return -EINVAL;
809
810 switch (mtt->mtt_type) {
811 case MTT_TYPE_WQE:
812 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
813 break;
814 case MTT_TYPE_CQE:
815 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
816 break;
817 case MTT_TYPE_SRQWQE:
818 bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
819 break;
820 case MTT_TYPE_IDX:
821 bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
822 break;
823 default:
824 dev_err(hr_dev->dev,
825 "Unsupport mtt type %d, write mtt failed\n",
826 mtt->mtt_type);
827 return -EINVAL;
828 }
829
830 while (npages > 0) {
831 chunk = min_t(int, bt_page_size / sizeof(u64), npages);
832
833 ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk,
834 page_list);
835 if (ret)
836 return ret;
837
838 npages -= chunk;
839 start_index += chunk;
840 page_list += chunk;
841 }
842
843 return 0;
844}
845
846int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
847 struct hns_roce_mtt *mtt, struct hns_roce_buf *buf)
848{
849 u64 *page_list;
850 int ret;
851 u32 i;
852
853 page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL);
854 if (!page_list)
855 return -ENOMEM;
856
857 for (i = 0; i < buf->npages; ++i) {
858 if (buf->nbufs == 1)
859 page_list[i] = buf->direct.map + (i << buf->page_shift);
860 else
861 page_list[i] = buf->page_list[i].map;
862
863 }
864 ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list);
865
866 kfree(page_list);
867
868 return ret;
869}
870
871int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)
872{
873 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
874 int ret;
875
876 ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap,
877 hr_dev->caps.num_mtpts,
878 hr_dev->caps.num_mtpts - 1,
879 hr_dev->caps.reserved_mrws, 0);
880 if (ret)
881 return ret;
882
883 ret = hns_roce_buddy_init(&mr_table->mtt_buddy,
884 ilog2(hr_dev->caps.num_mtt_segs));
885 if (ret)
886 goto err_buddy;
887
888 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {
889 ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy,
890 ilog2(hr_dev->caps.num_cqe_segs));
891 if (ret)
892 goto err_buddy_cqe;
893 }
894
895 if (hr_dev->caps.num_srqwqe_segs) {
896 ret = hns_roce_buddy_init(&mr_table->mtt_srqwqe_buddy,
897 ilog2(hr_dev->caps.num_srqwqe_segs));
898 if (ret)
899 goto err_buddy_srqwqe;
900 }
901
902 if (hr_dev->caps.num_idx_segs) {
903 ret = hns_roce_buddy_init(&mr_table->mtt_idx_buddy,
904 ilog2(hr_dev->caps.num_idx_segs));
905 if (ret)
906 goto err_buddy_idx;
907 }
908
909 return 0;
910
911err_buddy_idx:
912 if (hr_dev->caps.num_srqwqe_segs)
913 hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
914
915err_buddy_srqwqe:
916 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
917 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
918
919err_buddy_cqe:
920 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
921
922err_buddy:
923 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
924 return ret;
925}
926
927void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev)
928{
929 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
930
931 if (hr_dev->caps.num_idx_segs)
932 hns_roce_buddy_cleanup(&mr_table->mtt_idx_buddy);
933 if (hr_dev->caps.num_srqwqe_segs)
934 hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
935 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
936 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
937 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
938 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
939}
940
941struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)
942{
943 struct hns_roce_mr *mr;
944 int ret;
945
946 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
947 if (mr == NULL)
948 return ERR_PTR(-ENOMEM);
949
950 mr->type = MR_TYPE_DMA;
951
952
953 ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0,
954 ~0ULL, acc, 0, mr);
955 if (ret)
956 goto err_free;
957
958 ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr);
959 if (ret)
960 goto err_mr;
961
962 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
963 mr->umem = NULL;
964
965 return &mr->ibmr;
966
967err_mr:
968 hns_roce_mr_free(to_hr_dev(pd->device), mr);
969
970err_free:
971 kfree(mr);
972 return ERR_PTR(ret);
973}
974
975int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
976 struct hns_roce_mtt *mtt, struct ib_umem *umem)
977{
978 struct device *dev = hr_dev->dev;
979 struct scatterlist *sg;
980 unsigned int order;
981 int i, k, entry;
982 int npage = 0;
983 int ret = 0;
984 int len;
985 u64 page_addr;
986 u64 *pages;
987 u32 bt_page_size;
988 u32 n;
989
990 switch (mtt->mtt_type) {
991 case MTT_TYPE_WQE:
992 order = hr_dev->caps.mtt_ba_pg_sz;
993 break;
994 case MTT_TYPE_CQE:
995 order = hr_dev->caps.cqe_ba_pg_sz;
996 break;
997 case MTT_TYPE_SRQWQE:
998 order = hr_dev->caps.srqwqe_ba_pg_sz;
999 break;
1000 case MTT_TYPE_IDX:
1001 order = hr_dev->caps.idx_ba_pg_sz;
1002 break;
1003 default:
1004 dev_err(dev, "Unsupport mtt type %d, write mtt failed\n",
1005 mtt->mtt_type);
1006 return -EINVAL;
1007 }
1008
1009 bt_page_size = 1 << (order + PAGE_SHIFT);
1010
1011 pages = (u64 *) __get_free_pages(GFP_KERNEL, order);
1012 if (!pages)
1013 return -ENOMEM;
1014
1015 i = n = 0;
1016
1017 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
1018 len = sg_dma_len(sg) >> PAGE_SHIFT;
1019 for (k = 0; k < len; ++k) {
1020 page_addr =
1021 sg_dma_address(sg) + (k << umem->page_shift);
1022 if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) {
1023 if (page_addr & ((1 << mtt->page_shift) - 1)) {
1024 dev_err(dev, "page_addr 0x%llx is not page_shift %d alignment!\n",
1025 page_addr, mtt->page_shift);
1026 ret = -EINVAL;
1027 goto out;
1028 }
1029 pages[i++] = page_addr;
1030 }
1031 npage++;
1032 if (i == bt_page_size / sizeof(u64)) {
1033 ret = hns_roce_write_mtt(hr_dev, mtt, n, i,
1034 pages);
1035 if (ret)
1036 goto out;
1037 n += i;
1038 i = 0;
1039 }
1040 }
1041 }
1042
1043 if (i)
1044 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
1045
1046out:
1047 free_pages((unsigned long) pages, order);
1048 return ret;
1049}
1050
1051static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,
1052 struct hns_roce_mr *mr,
1053 struct ib_umem *umem)
1054{
1055 struct scatterlist *sg;
1056 int i = 0, j = 0, k;
1057 int entry;
1058 int len;
1059 u64 page_addr;
1060 u32 pbl_bt_sz;
1061
1062 if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0)
1063 return 0;
1064
1065 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
1066 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
1067 len = sg_dma_len(sg) >> PAGE_SHIFT;
1068 for (k = 0; k < len; ++k) {
1069 page_addr = sg_dma_address(sg) +
1070 (k << umem->page_shift);
1071
1072 if (!hr_dev->caps.pbl_hop_num) {
1073 mr->pbl_buf[i++] = page_addr >> 12;
1074 } else if (hr_dev->caps.pbl_hop_num == 1) {
1075 mr->pbl_buf[i++] = page_addr;
1076 } else {
1077 if (hr_dev->caps.pbl_hop_num == 2)
1078 mr->pbl_bt_l1[i][j] = page_addr;
1079 else if (hr_dev->caps.pbl_hop_num == 3)
1080 mr->pbl_bt_l2[i][j] = page_addr;
1081
1082 j++;
1083 if (j >= (pbl_bt_sz / 8)) {
1084 i++;
1085 j = 0;
1086 }
1087 }
1088 }
1089 }
1090
1091
1092 mb();
1093
1094 return 0;
1095}
1096
1097struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1098 u64 virt_addr, int access_flags,
1099 struct ib_udata *udata)
1100{
1101 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
1102 struct device *dev = hr_dev->dev;
1103 struct hns_roce_mr *mr;
1104 int bt_size;
1105 int ret;
1106 int n;
1107 int i;
1108
1109 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
1110 if (!mr)
1111 return ERR_PTR(-ENOMEM);
1112
1113 mr->umem = ib_umem_get(pd->uobject->context, start, length,
1114 access_flags, 0);
1115 if (IS_ERR(mr->umem)) {
1116 ret = PTR_ERR(mr->umem);
1117 goto err_free;
1118 }
1119
1120 n = ib_umem_page_count(mr->umem);
1121
1122 if (!hr_dev->caps.pbl_hop_num) {
1123 if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) {
1124 dev_err(dev,
1125 " MR len %lld err. MR is limited to 4G at most!\n",
1126 length);
1127 ret = -EINVAL;
1128 goto err_umem;
1129 }
1130 } else {
1131 u64 pbl_size = 1;
1132
1133 bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8;
1134 for (i = 0; i < hr_dev->caps.pbl_hop_num; i++)
1135 pbl_size *= bt_size;
1136 if (n > pbl_size) {
1137 dev_err(dev,
1138 " MR len %lld err. MR page num is limited to %lld!\n",
1139 length, pbl_size);
1140 ret = -EINVAL;
1141 goto err_umem;
1142 }
1143 }
1144
1145 mr->type = MR_TYPE_MR;
1146
1147 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length,
1148 access_flags, n, mr);
1149 if (ret)
1150 goto err_umem;
1151
1152 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
1153 if (ret)
1154 goto err_mr;
1155
1156 ret = hns_roce_mr_enable(hr_dev, mr);
1157 if (ret)
1158 goto err_mr;
1159
1160 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
1161
1162 return &mr->ibmr;
1163
1164err_mr:
1165 hns_roce_mr_free(hr_dev, mr);
1166
1167err_umem:
1168 ib_umem_release(mr->umem);
1169
1170err_free:
1171 kfree(mr);
1172 return ERR_PTR(ret);
1173}
1174
1175int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
1176 u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
1177 struct ib_udata *udata)
1178{
1179 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1180 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1181 struct hns_roce_cmd_mailbox *mailbox;
1182 struct device *dev = hr_dev->dev;
1183 unsigned long mtpt_idx;
1184 u32 pdn = 0;
1185 int npages;
1186 int ret;
1187
1188 if (!mr->enabled)
1189 return -EINVAL;
1190
1191 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1192 if (IS_ERR(mailbox))
1193 return PTR_ERR(mailbox);
1194
1195 mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);
1196 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0,
1197 HNS_ROCE_CMD_QUERY_MPT,
1198 HNS_ROCE_CMD_TIMEOUT_MSECS);
1199 if (ret)
1200 goto free_cmd_mbox;
1201
1202 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx);
1203 if (ret)
1204 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
1205
1206 mr->enabled = 0;
1207
1208 if (flags & IB_MR_REREG_PD)
1209 pdn = to_hr_pd(pd)->pdn;
1210
1211 if (flags & IB_MR_REREG_TRANS) {
1212 if (mr->size != ~0ULL) {
1213 npages = ib_umem_page_count(mr->umem);
1214
1215 if (hr_dev->caps.pbl_hop_num)
1216 hns_roce_mhop_free(hr_dev, mr);
1217 else
1218 dma_free_coherent(dev, npages * 8, mr->pbl_buf,
1219 mr->pbl_dma_addr);
1220 }
1221 ib_umem_release(mr->umem);
1222
1223 mr->umem = ib_umem_get(ibmr->uobject->context, start, length,
1224 mr_access_flags, 0);
1225 if (IS_ERR(mr->umem)) {
1226 ret = PTR_ERR(mr->umem);
1227 mr->umem = NULL;
1228 goto free_cmd_mbox;
1229 }
1230 npages = ib_umem_page_count(mr->umem);
1231
1232 if (hr_dev->caps.pbl_hop_num) {
1233 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
1234 if (ret)
1235 goto release_umem;
1236 } else {
1237 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
1238 &(mr->pbl_dma_addr),
1239 GFP_KERNEL);
1240 if (!mr->pbl_buf) {
1241 ret = -ENOMEM;
1242 goto release_umem;
1243 }
1244 }
1245 }
1246
1247 ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn,
1248 mr_access_flags, virt_addr,
1249 length, mailbox->buf);
1250 if (ret) {
1251 if (flags & IB_MR_REREG_TRANS)
1252 goto release_umem;
1253 else
1254 goto free_cmd_mbox;
1255 }
1256
1257 if (flags & IB_MR_REREG_TRANS) {
1258 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
1259 if (ret) {
1260 if (mr->size != ~0ULL) {
1261 npages = ib_umem_page_count(mr->umem);
1262
1263 if (hr_dev->caps.pbl_hop_num)
1264 hns_roce_mhop_free(hr_dev, mr);
1265 else
1266 dma_free_coherent(dev, npages * 8,
1267 mr->pbl_buf,
1268 mr->pbl_dma_addr);
1269 }
1270
1271 goto release_umem;
1272 }
1273 }
1274
1275 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx);
1276 if (ret) {
1277 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
1278 goto release_umem;
1279 }
1280
1281 mr->enabled = 1;
1282 if (flags & IB_MR_REREG_ACCESS)
1283 mr->access = mr_access_flags;
1284
1285 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1286
1287 return 0;
1288
1289release_umem:
1290 ib_umem_release(mr->umem);
1291
1292free_cmd_mbox:
1293 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1294
1295 return ret;
1296}
1297
1298int hns_roce_dereg_mr(struct ib_mr *ibmr)
1299{
1300 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1301 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1302 int ret = 0;
1303
1304 if (hr_dev->hw->dereg_mr) {
1305 ret = hr_dev->hw->dereg_mr(hr_dev, mr);
1306 } else {
1307 hns_roce_mr_free(hr_dev, mr);
1308
1309 if (mr->umem)
1310 ib_umem_release(mr->umem);
1311
1312 kfree(mr);
1313 }
1314
1315 return ret;
1316}
1317
1318struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1319 u32 max_num_sg)
1320{
1321 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
1322 struct device *dev = hr_dev->dev;
1323 struct hns_roce_mr *mr;
1324 u64 length;
1325 u32 page_size;
1326 int ret;
1327
1328 page_size = 1 << (hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT);
1329 length = max_num_sg * page_size;
1330
1331 if (mr_type != IB_MR_TYPE_MEM_REG)
1332 return ERR_PTR(-EINVAL);
1333
1334 if (max_num_sg > HNS_ROCE_FRMR_MAX_PA) {
1335 dev_err(dev, "max_num_sg larger than %d\n",
1336 HNS_ROCE_FRMR_MAX_PA);
1337 return ERR_PTR(-EINVAL);
1338 }
1339
1340 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1341 if (!mr)
1342 return ERR_PTR(-ENOMEM);
1343
1344 mr->type = MR_TYPE_FRMR;
1345
1346
1347 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, 0, length,
1348 0, max_num_sg, mr);
1349 if (ret)
1350 goto err_free;
1351
1352 ret = hns_roce_mr_enable(hr_dev, mr);
1353 if (ret)
1354 goto err_mr;
1355
1356 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
1357 mr->umem = NULL;
1358
1359 return &mr->ibmr;
1360
1361err_mr:
1362 hns_roce_mr_free(to_hr_dev(pd->device), mr);
1363
1364err_free:
1365 kfree(mr);
1366 return ERR_PTR(ret);
1367}
1368
1369static int hns_roce_set_page(struct ib_mr *ibmr, u64 addr)
1370{
1371 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1372
1373 mr->pbl_buf[mr->npages++] = cpu_to_le64(addr);
1374
1375 return 0;
1376}
1377
1378int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1379 unsigned int *sg_offset)
1380{
1381 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1382
1383 mr->npages = 0;
1384
1385 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page);
1386}
1387
1388static void hns_roce_mw_free(struct hns_roce_dev *hr_dev,
1389 struct hns_roce_mw *mw)
1390{
1391 struct device *dev = hr_dev->dev;
1392 int ret;
1393
1394 if (mw->enabled) {
1395 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mw->rkey)
1396 & (hr_dev->caps.num_mtpts - 1));
1397 if (ret)
1398 dev_warn(dev, "MW HW2SW_MPT failed (%d)\n", ret);
1399
1400 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
1401 key_to_hw_index(mw->rkey));
1402 }
1403
1404 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
1405 key_to_hw_index(mw->rkey), BITMAP_NO_RR);
1406}
1407
1408static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev,
1409 struct hns_roce_mw *mw)
1410{
1411 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
1412 struct hns_roce_cmd_mailbox *mailbox;
1413 struct device *dev = hr_dev->dev;
1414 unsigned long mtpt_idx = key_to_hw_index(mw->rkey);
1415 int ret;
1416
1417
1418 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
1419 if (ret)
1420 return ret;
1421
1422 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1423 if (IS_ERR(mailbox)) {
1424 ret = PTR_ERR(mailbox);
1425 goto err_table;
1426 }
1427
1428 ret = hr_dev->hw->mw_write_mtpt(mailbox->buf, mw);
1429 if (ret) {
1430 dev_err(dev, "MW write mtpt fail!\n");
1431 goto err_page;
1432 }
1433
1434 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,
1435 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
1436 if (ret) {
1437 dev_err(dev, "MW sw2hw_mpt failed (%d)\n", ret);
1438 goto err_page;
1439 }
1440
1441 mw->enabled = 1;
1442
1443 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1444
1445 return 0;
1446
1447err_page:
1448 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1449
1450err_table:
1451 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
1452
1453 return ret;
1454}
1455
1456struct ib_mw *hns_roce_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
1457 struct ib_udata *udata)
1458{
1459 struct hns_roce_dev *hr_dev = to_hr_dev(ib_pd->device);
1460 struct hns_roce_mw *mw;
1461 unsigned long index = 0;
1462 int ret;
1463
1464 mw = kmalloc(sizeof(*mw), GFP_KERNEL);
1465 if (!mw)
1466 return ERR_PTR(-ENOMEM);
1467
1468
1469 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
1470 if (ret)
1471 goto err_bitmap;
1472
1473 mw->rkey = hw_index_to_key(index);
1474
1475 mw->ibmw.rkey = mw->rkey;
1476 mw->ibmw.type = type;
1477 mw->pdn = to_hr_pd(ib_pd)->pdn;
1478 mw->pbl_hop_num = hr_dev->caps.pbl_hop_num;
1479 mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
1480 mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
1481
1482 ret = hns_roce_mw_enable(hr_dev, mw);
1483 if (ret)
1484 goto err_mw;
1485
1486 return &mw->ibmw;
1487
1488err_mw:
1489 hns_roce_mw_free(hr_dev, mw);
1490
1491err_bitmap:
1492 kfree(mw);
1493
1494 return ERR_PTR(ret);
1495}
1496
1497int hns_roce_dealloc_mw(struct ib_mw *ibmw)
1498{
1499 struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device);
1500 struct hns_roce_mw *mw = to_hr_mw(ibmw);
1501
1502 hns_roce_mw_free(hr_dev, mw);
1503 kfree(mw);
1504
1505 return 0;
1506}
1507
