1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/platform_device.h>
35#include <linux/vmalloc.h>
36#include <rdma/ib_umem.h>
37#include "hns_roce_device.h"
38#include "hns_roce_cmd.h"
39#include "hns_roce_hem.h"
40
41static u32 hw_index_to_key(unsigned long ind)
42{
43 return (u32)(ind >> 24) | (ind << 8);
44}
45
46unsigned long key_to_hw_index(u32 key)
47{
48 return (key << 24) | (key >> 8);
49}
50EXPORT_SYMBOL_GPL(key_to_hw_index);
51
52static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,
53 struct hns_roce_cmd_mailbox *mailbox,
54 unsigned long mpt_index)
55{
56 return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0,
57 HNS_ROCE_CMD_SW2HW_MPT,
58 HNS_ROCE_CMD_TIMEOUT_MSECS);
59}
60
61int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
62 struct hns_roce_cmd_mailbox *mailbox,
63 unsigned long mpt_index)
64{
65 return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0,
66 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,
67 HNS_ROCE_CMD_TIMEOUT_MSECS);
68}
69EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt);
70
71static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
72 unsigned long *seg)
73{
74 int o;
75 u32 m;
76
77 spin_lock(&buddy->lock);
78
79 for (o = order; o <= buddy->max_order; ++o) {
80 if (buddy->num_free[o]) {
81 m = 1 << (buddy->max_order - o);
82 *seg = find_first_bit(buddy->bits[o], m);
83 if (*seg < m)
84 goto found;
85 }
86 }
87 spin_unlock(&buddy->lock);
88 return -1;
89
90 found:
91 clear_bit(*seg, buddy->bits[o]);
92 --buddy->num_free[o];
93
94 while (o > order) {
95 --o;
96 *seg <<= 1;
97 set_bit(*seg ^ 1, buddy->bits[o]);
98 ++buddy->num_free[o];
99 }
100
101 spin_unlock(&buddy->lock);
102
103 *seg <<= order;
104 return 0;
105}
106
107static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg,
108 int order)
109{
110 seg >>= order;
111
112 spin_lock(&buddy->lock);
113
114 while (test_bit(seg ^ 1, buddy->bits[order])) {
115 clear_bit(seg ^ 1, buddy->bits[order]);
116 --buddy->num_free[order];
117 seg >>= 1;
118 ++order;
119 }
120
121 set_bit(seg, buddy->bits[order]);
122 ++buddy->num_free[order];
123
124 spin_unlock(&buddy->lock);
125}
126
127static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
128{
129 int i, s;
130
131 buddy->max_order = max_order;
132 spin_lock_init(&buddy->lock);
133 buddy->bits = kcalloc(buddy->max_order + 1,
134 sizeof(*buddy->bits),
135 GFP_KERNEL);
136 buddy->num_free = kcalloc(buddy->max_order + 1,
137 sizeof(*buddy->num_free),
138 GFP_KERNEL);
139 if (!buddy->bits || !buddy->num_free)
140 goto err_out;
141
142 for (i = 0; i <= buddy->max_order; ++i) {
143 s = BITS_TO_LONGS(1 << (buddy->max_order - i));
144 buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL |
145 __GFP_NOWARN);
146 if (!buddy->bits[i]) {
147 buddy->bits[i] = vzalloc(array_size(s, sizeof(long)));
148 if (!buddy->bits[i])
149 goto err_out_free;
150 }
151 }
152
153 set_bit(0, buddy->bits[buddy->max_order]);
154 buddy->num_free[buddy->max_order] = 1;
155
156 return 0;
157
158err_out_free:
159 for (i = 0; i <= buddy->max_order; ++i)
160 kvfree(buddy->bits[i]);
161
162err_out:
163 kfree(buddy->bits);
164 kfree(buddy->num_free);
165 return -ENOMEM;
166}
167
168static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)
169{
170 int i;
171
172 for (i = 0; i <= buddy->max_order; ++i)
173 kvfree(buddy->bits[i]);
174
175 kfree(buddy->bits);
176 kfree(buddy->num_free);
177}
178
179static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order,
180 unsigned long *seg, u32 mtt_type)
181{
182 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
183 struct hns_roce_hem_table *table;
184 struct hns_roce_buddy *buddy;
185 int ret;
186
187 if (mtt_type == MTT_TYPE_WQE) {
188 buddy = &mr_table->mtt_buddy;
189 table = &mr_table->mtt_table;
190 } else {
191 buddy = &mr_table->mtt_cqe_buddy;
192 table = &mr_table->mtt_cqe_table;
193 }
194
195 ret = hns_roce_buddy_alloc(buddy, order, seg);
196 if (ret == -1)
197 return -1;
198
199 if (hns_roce_table_get_range(hr_dev, table, *seg,
200 *seg + (1 << order) - 1)) {
201 hns_roce_buddy_free(buddy, *seg, order);
202 return -1;
203 }
204
205 return 0;
206}
207
208int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
209 struct hns_roce_mtt *mtt)
210{
211 int ret;
212 int i;
213
214
215 if (!npages) {
216 mtt->order = -1;
217 mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT;
218 return 0;
219 }
220
221
222 mtt->page_shift = page_shift;
223
224
225 for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages;
226 i <<= 1)
227 ++mtt->order;
228
229
230 ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg,
231 mtt->mtt_type);
232 if (ret == -1)
233 return -ENOMEM;
234
235 return 0;
236}
237
238void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)
239{
240 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
241
242 if (mtt->order < 0)
243 return;
244
245 if (mtt->mtt_type == MTT_TYPE_WQE) {
246 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,
247 mtt->order);
248 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,
249 mtt->first_seg,
250 mtt->first_seg + (1 << mtt->order) - 1);
251 } else {
252 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,
253 mtt->order);
254 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,
255 mtt->first_seg,
256 mtt->first_seg + (1 << mtt->order) - 1);
257 }
258}
259EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);
260
261static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
262 struct hns_roce_mr *mr, int err_loop_index,
263 int loop_i, int loop_j)
264{
265 struct device *dev = hr_dev->dev;
266 u32 mhop_num;
267 u32 pbl_bt_sz;
268 u64 bt_idx;
269 int i, j;
270
271 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
272 mhop_num = hr_dev->caps.pbl_hop_num;
273
274 i = loop_i;
275 if (mhop_num == 3 && err_loop_index == 2) {
276 for (; i >= 0; i--) {
277 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
278 mr->pbl_l1_dma_addr[i]);
279
280 for (j = 0; j < pbl_bt_sz / 8; j++) {
281 if (i == loop_i && j >= loop_j)
282 break;
283
284 bt_idx = i * pbl_bt_sz / 8 + j;
285 dma_free_coherent(dev, pbl_bt_sz,
286 mr->pbl_bt_l2[bt_idx],
287 mr->pbl_l2_dma_addr[bt_idx]);
288 }
289 }
290 } else if (mhop_num == 3 && err_loop_index == 1) {
291 for (i -= 1; i >= 0; i--) {
292 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
293 mr->pbl_l1_dma_addr[i]);
294
295 for (j = 0; j < pbl_bt_sz / 8; j++) {
296 bt_idx = i * pbl_bt_sz / 8 + j;
297 dma_free_coherent(dev, pbl_bt_sz,
298 mr->pbl_bt_l2[bt_idx],
299 mr->pbl_l2_dma_addr[bt_idx]);
300 }
301 }
302 } else if (mhop_num == 2 && err_loop_index == 1) {
303 for (i -= 1; i >= 0; i--)
304 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
305 mr->pbl_l1_dma_addr[i]);
306 } else {
307 dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.",
308 mhop_num, err_loop_index);
309 return;
310 }
311
312 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr);
313 mr->pbl_bt_l0 = NULL;
314 mr->pbl_l0_dma_addr = 0;
315}
316
317
318static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages,
319 struct hns_roce_mr *mr)
320{
321 struct device *dev = hr_dev->dev;
322 int mr_alloc_done = 0;
323 int npages_allocated;
324 int i = 0, j = 0;
325 u32 pbl_bt_sz;
326 u32 mhop_num;
327 u64 pbl_last_bt_num;
328 u64 pbl_bt_cnt = 0;
329 u64 bt_idx;
330 u64 size;
331
332 mhop_num = hr_dev->caps.pbl_hop_num;
333 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
334 pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);
335
336 if (mhop_num == HNS_ROCE_HOP_NUM_0)
337 return 0;
338
339
340 if (mhop_num == 1) {
341 if (npages > pbl_bt_sz / 8) {
342 dev_err(dev, "npages %d is larger than buf_pg_sz!",
343 npages);
344 return -EINVAL;
345 }
346 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
347 &(mr->pbl_dma_addr),
348 GFP_KERNEL);
349 if (!mr->pbl_buf)
350 return -ENOMEM;
351
352 mr->pbl_size = npages;
353 mr->pbl_ba = mr->pbl_dma_addr;
354 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num;
355 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
356 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
357 return 0;
358 }
359
360 mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8,
361 sizeof(*mr->pbl_l1_dma_addr),
362 GFP_KERNEL);
363 if (!mr->pbl_l1_dma_addr)
364 return -ENOMEM;
365
366 mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1),
367 GFP_KERNEL);
368 if (!mr->pbl_bt_l1)
369 goto err_kcalloc_bt_l1;
370
371 if (mhop_num == 3) {
372 mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num,
373 sizeof(*mr->pbl_l2_dma_addr),
374 GFP_KERNEL);
375 if (!mr->pbl_l2_dma_addr)
376 goto err_kcalloc_l2_dma;
377
378 mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num,
379 sizeof(*mr->pbl_bt_l2),
380 GFP_KERNEL);
381 if (!mr->pbl_bt_l2)
382 goto err_kcalloc_bt_l2;
383 }
384
385
386 mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz,
387 &(mr->pbl_l0_dma_addr),
388 GFP_KERNEL);
389 if (!mr->pbl_bt_l0)
390 goto err_dma_alloc_l0;
391
392 if (mhop_num == 2) {
393
394 for (i = 0; i < pbl_bt_sz / 8; i++) {
395 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
396 size = pbl_bt_sz;
397 } else {
398 npages_allocated = i * (pbl_bt_sz / 8);
399 size = (npages - npages_allocated) * 8;
400 }
401 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size,
402 &(mr->pbl_l1_dma_addr[i]),
403 GFP_KERNEL);
404 if (!mr->pbl_bt_l1[i]) {
405 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
406 goto err_dma_alloc_l0;
407 }
408
409 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
410
411 pbl_bt_cnt++;
412 if (pbl_bt_cnt >= pbl_last_bt_num)
413 break;
414 }
415 } else if (mhop_num == 3) {
416
417 for (i = 0; i < pbl_bt_sz / 8; i++) {
418 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz,
419 &(mr->pbl_l1_dma_addr[i]),
420 GFP_KERNEL);
421 if (!mr->pbl_bt_l1[i]) {
422 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
423 goto err_dma_alloc_l0;
424 }
425
426 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
427
428 for (j = 0; j < pbl_bt_sz / 8; j++) {
429 bt_idx = i * pbl_bt_sz / 8 + j;
430
431 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
432 size = pbl_bt_sz;
433 } else {
434 npages_allocated = bt_idx *
435 (pbl_bt_sz / 8);
436 size = (npages - npages_allocated) * 8;
437 }
438 mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent(
439 dev, size,
440 &(mr->pbl_l2_dma_addr[bt_idx]),
441 GFP_KERNEL);
442 if (!mr->pbl_bt_l2[bt_idx]) {
443 hns_roce_loop_free(hr_dev, mr, 2, i, j);
444 goto err_dma_alloc_l0;
445 }
446
447 *(mr->pbl_bt_l1[i] + j) =
448 mr->pbl_l2_dma_addr[bt_idx];
449
450 pbl_bt_cnt++;
451 if (pbl_bt_cnt >= pbl_last_bt_num) {
452 mr_alloc_done = 1;
453 break;
454 }
455 }
456
457 if (mr_alloc_done)
458 break;
459 }
460 }
461
462 mr->l0_chunk_last_num = i + 1;
463 if (mhop_num == 3)
464 mr->l1_chunk_last_num = j + 1;
465
466 mr->pbl_size = npages;
467 mr->pbl_ba = mr->pbl_l0_dma_addr;
468 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num;
469 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
470 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
471
472 return 0;
473
474err_dma_alloc_l0:
475 kfree(mr->pbl_bt_l2);
476 mr->pbl_bt_l2 = NULL;
477
478err_kcalloc_bt_l2:
479 kfree(mr->pbl_l2_dma_addr);
480 mr->pbl_l2_dma_addr = NULL;
481
482err_kcalloc_l2_dma:
483 kfree(mr->pbl_bt_l1);
484 mr->pbl_bt_l1 = NULL;
485
486err_kcalloc_bt_l1:
487 kfree(mr->pbl_l1_dma_addr);
488 mr->pbl_l1_dma_addr = NULL;
489
490 return -ENOMEM;
491}
492
493static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
494 u64 size, u32 access, int npages,
495 struct hns_roce_mr *mr)
496{
497 struct device *dev = hr_dev->dev;
498 unsigned long index = 0;
499 int ret = 0;
500
501
502 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
503 if (ret == -1)
504 return -ENOMEM;
505
506 mr->iova = iova;
507 mr->size = size;
508 mr->pd = pd;
509 mr->access = access;
510 mr->enabled = 0;
511 mr->key = hw_index_to_key(index);
512
513 if (size == ~0ull) {
514 mr->type = MR_TYPE_DMA;
515 mr->pbl_buf = NULL;
516 mr->pbl_dma_addr = 0;
517
518 mr->pbl_bt_l2 = NULL;
519 mr->pbl_bt_l1 = NULL;
520 mr->pbl_bt_l0 = NULL;
521 mr->pbl_l2_dma_addr = NULL;
522 mr->pbl_l1_dma_addr = NULL;
523 mr->pbl_l0_dma_addr = 0;
524 } else {
525 mr->type = MR_TYPE_MR;
526 if (!hr_dev->caps.pbl_hop_num) {
527 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
528 &(mr->pbl_dma_addr),
529 GFP_KERNEL);
530 if (!mr->pbl_buf)
531 return -ENOMEM;
532 } else {
533 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
534 }
535 }
536
537 return ret;
538}
539
540static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
541 struct hns_roce_mr *mr)
542{
543 struct device *dev = hr_dev->dev;
544 int npages_allocated;
545 int npages;
546 int i, j;
547 u32 pbl_bt_sz;
548 u32 mhop_num;
549 u64 bt_idx;
550
551 npages = ib_umem_page_count(mr->umem);
552 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
553 mhop_num = hr_dev->caps.pbl_hop_num;
554
555 if (mhop_num == HNS_ROCE_HOP_NUM_0)
556 return;
557
558
559 if (mhop_num == 1) {
560 dma_free_coherent(dev, (unsigned int)(npages * 8),
561 mr->pbl_buf, mr->pbl_dma_addr);
562 return;
563 }
564
565 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0,
566 mr->pbl_l0_dma_addr);
567
568 if (mhop_num == 2) {
569 for (i = 0; i < mr->l0_chunk_last_num; i++) {
570 if (i == mr->l0_chunk_last_num - 1) {
571 npages_allocated = i * (pbl_bt_sz / 8);
572
573 dma_free_coherent(dev,
574 (npages - npages_allocated) * 8,
575 mr->pbl_bt_l1[i],
576 mr->pbl_l1_dma_addr[i]);
577
578 break;
579 }
580
581 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
582 mr->pbl_l1_dma_addr[i]);
583 }
584 } else if (mhop_num == 3) {
585 for (i = 0; i < mr->l0_chunk_last_num; i++) {
586 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
587 mr->pbl_l1_dma_addr[i]);
588
589 for (j = 0; j < pbl_bt_sz / 8; j++) {
590 bt_idx = i * (pbl_bt_sz / 8) + j;
591
592 if ((i == mr->l0_chunk_last_num - 1)
593 && j == mr->l1_chunk_last_num - 1) {
594 npages_allocated = bt_idx *
595 (pbl_bt_sz / 8);
596
597 dma_free_coherent(dev,
598 (npages - npages_allocated) * 8,
599 mr->pbl_bt_l2[bt_idx],
600 mr->pbl_l2_dma_addr[bt_idx]);
601
602 break;
603 }
604
605 dma_free_coherent(dev, pbl_bt_sz,
606 mr->pbl_bt_l2[bt_idx],
607 mr->pbl_l2_dma_addr[bt_idx]);
608 }
609 }
610 }
611
612 kfree(mr->pbl_bt_l1);
613 kfree(mr->pbl_l1_dma_addr);
614 mr->pbl_bt_l1 = NULL;
615 mr->pbl_l1_dma_addr = NULL;
616 if (mhop_num == 3) {
617 kfree(mr->pbl_bt_l2);
618 kfree(mr->pbl_l2_dma_addr);
619 mr->pbl_bt_l2 = NULL;
620 mr->pbl_l2_dma_addr = NULL;
621 }
622}
623
624static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
625 struct hns_roce_mr *mr)
626{
627 struct device *dev = hr_dev->dev;
628 int npages = 0;
629 int ret;
630
631 if (mr->enabled) {
632 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
633 & (hr_dev->caps.num_mtpts - 1));
634 if (ret)
635 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
636 }
637
638 if (mr->size != ~0ULL) {
639 npages = ib_umem_page_count(mr->umem);
640
641 if (!hr_dev->caps.pbl_hop_num)
642 dma_free_coherent(dev, (unsigned int)(npages * 8),
643 mr->pbl_buf, mr->pbl_dma_addr);
644 else
645 hns_roce_mhop_free(hr_dev, mr);
646 }
647
648 if (mr->enabled)
649 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
650 key_to_hw_index(mr->key));
651
652 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
653 key_to_hw_index(mr->key), BITMAP_NO_RR);
654}
655
656static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
657 struct hns_roce_mr *mr)
658{
659 int ret;
660 unsigned long mtpt_idx = key_to_hw_index(mr->key);
661 struct device *dev = hr_dev->dev;
662 struct hns_roce_cmd_mailbox *mailbox;
663 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
664
665
666 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
667 if (ret)
668 return ret;
669
670
671 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
672 if (IS_ERR(mailbox)) {
673 ret = PTR_ERR(mailbox);
674 goto err_table;
675 }
676
677 ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx);
678 if (ret) {
679 dev_err(dev, "Write mtpt fail!\n");
680 goto err_page;
681 }
682
683 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,
684 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
685 if (ret) {
686 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
687 goto err_page;
688 }
689
690 mr->enabled = 1;
691 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
692
693 return 0;
694
695err_page:
696 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
697
698err_table:
699 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
700 return ret;
701}
702
703static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev,
704 struct hns_roce_mtt *mtt, u32 start_index,
705 u32 npages, u64 *page_list)
706{
707 struct hns_roce_hem_table *table;
708 dma_addr_t dma_handle;
709 __le64 *mtts;
710 u32 s = start_index * sizeof(u64);
711 u32 bt_page_size;
712 u32 i;
713
714 if (mtt->mtt_type == MTT_TYPE_WQE)
715 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
716 else
717 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
718
719
720 if (start_index / (bt_page_size / sizeof(u64)) !=
721 (start_index + npages - 1) / (bt_page_size / sizeof(u64)))
722 return -EINVAL;
723
724 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
725 return -EINVAL;
726
727 if (mtt->mtt_type == MTT_TYPE_WQE)
728 table = &hr_dev->mr_table.mtt_table;
729 else
730 table = &hr_dev->mr_table.mtt_cqe_table;
731
732 mtts = hns_roce_table_find(hr_dev, table,
733 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz,
734 &dma_handle);
735 if (!mtts)
736 return -ENOMEM;
737
738
739 for (i = 0; i < npages; ++i) {
740 if (!hr_dev->caps.mtt_hop_num)
741 mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT);
742 else
743 mtts[i] = cpu_to_le64(page_list[i]);
744 }
745
746 return 0;
747}
748
749static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev,
750 struct hns_roce_mtt *mtt, u32 start_index,
751 u32 npages, u64 *page_list)
752{
753 int chunk;
754 int ret;
755 u32 bt_page_size;
756
757 if (mtt->order < 0)
758 return -EINVAL;
759
760 if (mtt->mtt_type == MTT_TYPE_WQE)
761 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
762 else
763 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
764
765 while (npages > 0) {
766 chunk = min_t(int, bt_page_size / sizeof(u64), npages);
767
768 ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk,
769 page_list);
770 if (ret)
771 return ret;
772
773 npages -= chunk;
774 start_index += chunk;
775 page_list += chunk;
776 }
777
778 return 0;
779}
780
781int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
782 struct hns_roce_mtt *mtt, struct hns_roce_buf *buf)
783{
784 u64 *page_list;
785 int ret;
786 u32 i;
787
788 page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL);
789 if (!page_list)
790 return -ENOMEM;
791
792 for (i = 0; i < buf->npages; ++i) {
793 if (buf->nbufs == 1)
794 page_list[i] = buf->direct.map + (i << buf->page_shift);
795 else
796 page_list[i] = buf->page_list[i].map;
797
798 }
799 ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list);
800
801 kfree(page_list);
802
803 return ret;
804}
805
806int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)
807{
808 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
809 int ret;
810
811 ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap,
812 hr_dev->caps.num_mtpts,
813 hr_dev->caps.num_mtpts - 1,
814 hr_dev->caps.reserved_mrws, 0);
815 if (ret)
816 return ret;
817
818 ret = hns_roce_buddy_init(&mr_table->mtt_buddy,
819 ilog2(hr_dev->caps.num_mtt_segs));
820 if (ret)
821 goto err_buddy;
822
823 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {
824 ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy,
825 ilog2(hr_dev->caps.num_cqe_segs));
826 if (ret)
827 goto err_buddy_cqe;
828 }
829 return 0;
830
831err_buddy_cqe:
832 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
833
834err_buddy:
835 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
836 return ret;
837}
838
839void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev)
840{
841 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
842
843 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
844 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
845 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
846 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
847}
848
849struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)
850{
851 struct hns_roce_mr *mr;
852 int ret;
853
854 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
855 if (mr == NULL)
856 return ERR_PTR(-ENOMEM);
857
858
859 ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0,
860 ~0ULL, acc, 0, mr);
861 if (ret)
862 goto err_free;
863
864 ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr);
865 if (ret)
866 goto err_mr;
867
868 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
869 mr->umem = NULL;
870
871 return &mr->ibmr;
872
873err_mr:
874 hns_roce_mr_free(to_hr_dev(pd->device), mr);
875
876err_free:
877 kfree(mr);
878 return ERR_PTR(ret);
879}
880
881int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
882 struct hns_roce_mtt *mtt, struct ib_umem *umem)
883{
884 struct device *dev = hr_dev->dev;
885 struct scatterlist *sg;
886 unsigned int order;
887 int i, k, entry;
888 int npage = 0;
889 int ret = 0;
890 int len;
891 u64 page_addr;
892 u64 *pages;
893 u32 bt_page_size;
894 u32 n;
895
896 order = mtt->mtt_type == MTT_TYPE_WQE ? hr_dev->caps.mtt_ba_pg_sz :
897 hr_dev->caps.cqe_ba_pg_sz;
898 bt_page_size = 1 << (order + PAGE_SHIFT);
899
900 pages = (u64 *) __get_free_pages(GFP_KERNEL, order);
901 if (!pages)
902 return -ENOMEM;
903
904 i = n = 0;
905
906 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
907 len = sg_dma_len(sg) >> PAGE_SHIFT;
908 for (k = 0; k < len; ++k) {
909 page_addr =
910 sg_dma_address(sg) + (k << umem->page_shift);
911 if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) {
912 if (page_addr & ((1 << mtt->page_shift) - 1)) {
913 dev_err(dev, "page_addr 0x%llx is not page_shift %d alignment!\n",
914 page_addr, mtt->page_shift);
915 ret = -EINVAL;
916 goto out;
917 }
918 pages[i++] = page_addr;
919 }
920 npage++;
921 if (i == bt_page_size / sizeof(u64)) {
922 ret = hns_roce_write_mtt(hr_dev, mtt, n, i,
923 pages);
924 if (ret)
925 goto out;
926 n += i;
927 i = 0;
928 }
929 }
930 }
931
932 if (i)
933 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
934
935out:
936 free_pages((unsigned long) pages, order);
937 return ret;
938}
939
940static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,
941 struct hns_roce_mr *mr,
942 struct ib_umem *umem)
943{
944 struct scatterlist *sg;
945 int i = 0, j = 0, k;
946 int entry;
947 int len;
948 u64 page_addr;
949 u32 pbl_bt_sz;
950
951 if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0)
952 return 0;
953
954 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
955 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
956 len = sg_dma_len(sg) >> PAGE_SHIFT;
957 for (k = 0; k < len; ++k) {
958 page_addr = sg_dma_address(sg) +
959 (k << umem->page_shift);
960
961 if (!hr_dev->caps.pbl_hop_num) {
962 mr->pbl_buf[i++] = page_addr >> 12;
963 } else if (hr_dev->caps.pbl_hop_num == 1) {
964 mr->pbl_buf[i++] = page_addr;
965 } else {
966 if (hr_dev->caps.pbl_hop_num == 2)
967 mr->pbl_bt_l1[i][j] = page_addr;
968 else if (hr_dev->caps.pbl_hop_num == 3)
969 mr->pbl_bt_l2[i][j] = page_addr;
970
971 j++;
972 if (j >= (pbl_bt_sz / 8)) {
973 i++;
974 j = 0;
975 }
976 }
977 }
978 }
979
980
981 mb();
982
983 return 0;
984}
985
986struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
987 u64 virt_addr, int access_flags,
988 struct ib_udata *udata)
989{
990 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
991 struct device *dev = hr_dev->dev;
992 struct hns_roce_mr *mr;
993 int bt_size;
994 int ret;
995 int n;
996 int i;
997
998 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
999 if (!mr)
1000 return ERR_PTR(-ENOMEM);
1001
1002 mr->umem = ib_umem_get(pd->uobject->context, start, length,
1003 access_flags, 0);
1004 if (IS_ERR(mr->umem)) {
1005 ret = PTR_ERR(mr->umem);
1006 goto err_free;
1007 }
1008
1009 n = ib_umem_page_count(mr->umem);
1010
1011 if (!hr_dev->caps.pbl_hop_num) {
1012 if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) {
1013 dev_err(dev,
1014 " MR len %lld err. MR is limited to 4G at most!\n",
1015 length);
1016 ret = -EINVAL;
1017 goto err_umem;
1018 }
1019 } else {
1020 u64 pbl_size = 1;
1021
1022 bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8;
1023 for (i = 0; i < hr_dev->caps.pbl_hop_num; i++)
1024 pbl_size *= bt_size;
1025 if (n > pbl_size) {
1026 dev_err(dev,
1027 " MR len %lld err. MR page num is limited to %lld!\n",
1028 length, pbl_size);
1029 ret = -EINVAL;
1030 goto err_umem;
1031 }
1032 }
1033
1034 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length,
1035 access_flags, n, mr);
1036 if (ret)
1037 goto err_umem;
1038
1039 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
1040 if (ret)
1041 goto err_mr;
1042
1043 ret = hns_roce_mr_enable(hr_dev, mr);
1044 if (ret)
1045 goto err_mr;
1046
1047 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
1048
1049 return &mr->ibmr;
1050
1051err_mr:
1052 hns_roce_mr_free(hr_dev, mr);
1053
1054err_umem:
1055 ib_umem_release(mr->umem);
1056
1057err_free:
1058 kfree(mr);
1059 return ERR_PTR(ret);
1060}
1061
1062int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
1063 u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
1064 struct ib_udata *udata)
1065{
1066 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1067 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1068 struct hns_roce_cmd_mailbox *mailbox;
1069 struct device *dev = hr_dev->dev;
1070 unsigned long mtpt_idx;
1071 u32 pdn = 0;
1072 int npages;
1073 int ret;
1074
1075 if (!mr->enabled)
1076 return -EINVAL;
1077
1078 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1079 if (IS_ERR(mailbox))
1080 return PTR_ERR(mailbox);
1081
1082 mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);
1083 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0,
1084 HNS_ROCE_CMD_QUERY_MPT,
1085 HNS_ROCE_CMD_TIMEOUT_MSECS);
1086 if (ret)
1087 goto free_cmd_mbox;
1088
1089 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx);
1090 if (ret)
1091 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
1092
1093 mr->enabled = 0;
1094
1095 if (flags & IB_MR_REREG_PD)
1096 pdn = to_hr_pd(pd)->pdn;
1097
1098 if (flags & IB_MR_REREG_TRANS) {
1099 if (mr->size != ~0ULL) {
1100 npages = ib_umem_page_count(mr->umem);
1101
1102 if (hr_dev->caps.pbl_hop_num)
1103 hns_roce_mhop_free(hr_dev, mr);
1104 else
1105 dma_free_coherent(dev, npages * 8, mr->pbl_buf,
1106 mr->pbl_dma_addr);
1107 }
1108 ib_umem_release(mr->umem);
1109
1110 mr->umem = ib_umem_get(ibmr->uobject->context, start, length,
1111 mr_access_flags, 0);
1112 if (IS_ERR(mr->umem)) {
1113 ret = PTR_ERR(mr->umem);
1114 mr->umem = NULL;
1115 goto free_cmd_mbox;
1116 }
1117 npages = ib_umem_page_count(mr->umem);
1118
1119 if (hr_dev->caps.pbl_hop_num) {
1120 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
1121 if (ret)
1122 goto release_umem;
1123 } else {
1124 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
1125 &(mr->pbl_dma_addr),
1126 GFP_KERNEL);
1127 if (!mr->pbl_buf) {
1128 ret = -ENOMEM;
1129 goto release_umem;
1130 }
1131 }
1132 }
1133
1134 ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn,
1135 mr_access_flags, virt_addr,
1136 length, mailbox->buf);
1137 if (ret) {
1138 if (flags & IB_MR_REREG_TRANS)
1139 goto release_umem;
1140 else
1141 goto free_cmd_mbox;
1142 }
1143
1144 if (flags & IB_MR_REREG_TRANS) {
1145 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
1146 if (ret) {
1147 if (mr->size != ~0ULL) {
1148 npages = ib_umem_page_count(mr->umem);
1149
1150 if (hr_dev->caps.pbl_hop_num)
1151 hns_roce_mhop_free(hr_dev, mr);
1152 else
1153 dma_free_coherent(dev, npages * 8,
1154 mr->pbl_buf,
1155 mr->pbl_dma_addr);
1156 }
1157
1158 goto release_umem;
1159 }
1160 }
1161
1162 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx);
1163 if (ret) {
1164 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
1165 goto release_umem;
1166 }
1167
1168 mr->enabled = 1;
1169 if (flags & IB_MR_REREG_ACCESS)
1170 mr->access = mr_access_flags;
1171
1172 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1173
1174 return 0;
1175
1176release_umem:
1177 ib_umem_release(mr->umem);
1178
1179free_cmd_mbox:
1180 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1181
1182 return ret;
1183}
1184
1185int hns_roce_dereg_mr(struct ib_mr *ibmr)
1186{
1187 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1188 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1189 int ret = 0;
1190
1191 if (hr_dev->hw->dereg_mr) {
1192 ret = hr_dev->hw->dereg_mr(hr_dev, mr);
1193 } else {
1194 hns_roce_mr_free(hr_dev, mr);
1195
1196 if (mr->umem)
1197 ib_umem_release(mr->umem);
1198
1199 kfree(mr);
1200 }
1201
1202 return ret;
1203}
1204
