1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/platform_device.h>
35#include "hns_roce_device.h"
36#include "hns_roce_hem.h"
37#include "hns_roce_common.h"
38
39#define DMA_ADDR_T_SHIFT 12
40#define BT_BA_SHIFT 32
41
42bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
43{
44 if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
45 (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
46 (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
47 (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
48 (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
49 (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))
50 return true;
51
52 return false;
53}
54EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
55
56static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
57 u32 bt_chunk_num)
58{
59 int i;
60
61 for (i = 0; i < bt_chunk_num; i++)
62 if (hem[start_idx + i])
63 return false;
64
65 return true;
66}
67
68static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
69{
70 int i;
71
72 for (i = 0; i < bt_chunk_num; i++)
73 if (bt[start_idx + i])
74 return false;
75
76 return true;
77}
78
79static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
80{
81 if (check_whether_bt_num_3(table_type, hop_num))
82 return 3;
83 else if (check_whether_bt_num_2(table_type, hop_num))
84 return 2;
85 else if (check_whether_bt_num_1(table_type, hop_num))
86 return 1;
87 else
88 return 0;
89}
90
91int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
92 struct hns_roce_hem_table *table, unsigned long *obj,
93 struct hns_roce_hem_mhop *mhop)
94{
95 struct device *dev = hr_dev->dev;
96 u32 chunk_ba_num;
97 u32 table_idx;
98 u32 bt_num;
99 u32 chunk_size;
100
101 switch (table->type) {
102 case HEM_TYPE_QPC:
103 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
104 + PAGE_SHIFT);
105 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
106 + PAGE_SHIFT);
107 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
108 mhop->hop_num = hr_dev->caps.qpc_hop_num;
109 break;
110 case HEM_TYPE_MTPT:
111 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
112 + PAGE_SHIFT);
113 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
114 + PAGE_SHIFT);
115 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
116 mhop->hop_num = hr_dev->caps.mpt_hop_num;
117 break;
118 case HEM_TYPE_CQC:
119 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
120 + PAGE_SHIFT);
121 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
122 + PAGE_SHIFT);
123 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
124 mhop->hop_num = hr_dev->caps.cqc_hop_num;
125 break;
126 case HEM_TYPE_SRQC:
127 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
128 + PAGE_SHIFT);
129 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
130 + PAGE_SHIFT);
131 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
132 mhop->hop_num = hr_dev->caps.srqc_hop_num;
133 break;
134 case HEM_TYPE_MTT:
135 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
136 + PAGE_SHIFT);
137 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
138 + PAGE_SHIFT);
139 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
140 mhop->hop_num = hr_dev->caps.mtt_hop_num;
141 break;
142 case HEM_TYPE_CQE:
143 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
144 + PAGE_SHIFT);
145 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
146 + PAGE_SHIFT);
147 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
148 mhop->hop_num = hr_dev->caps.cqe_hop_num;
149 break;
150 default:
151 dev_err(dev, "Table %d not support multi-hop addressing!\n",
152 table->type);
153 return -EINVAL;
154 }
155
156 if (!obj)
157 return 0;
158
159
160
161
162
163 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
164 chunk_ba_num = mhop->bt_chunk_size / 8;
165 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
166 mhop->bt_chunk_size;
167 table_idx = (*obj & (table->num_obj - 1)) /
168 (chunk_size / table->obj_size);
169 switch (bt_num) {
170 case 3:
171 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
172 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
173 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
174 break;
175 case 2:
176 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
177 mhop->l0_idx = table_idx / chunk_ba_num;
178 break;
179 case 1:
180 mhop->l0_idx = table_idx;
181 break;
182 default:
183 dev_err(dev, "Table %d not support hop_num = %d!\n",
184 table->type, mhop->hop_num);
185 return -EINVAL;
186 }
187 if (mhop->l0_idx >= mhop->ba_l0_num)
188 mhop->l0_idx %= mhop->ba_l0_num;
189
190 return 0;
191}
192EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
193
194static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
195 int npages,
196 unsigned long hem_alloc_size,
197 gfp_t gfp_mask)
198{
199 struct hns_roce_hem_chunk *chunk = NULL;
200 struct hns_roce_hem *hem;
201 struct scatterlist *mem;
202 int order;
203 void *buf;
204
205 WARN_ON(gfp_mask & __GFP_HIGHMEM);
206
207 hem = kmalloc(sizeof(*hem),
208 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
209 if (!hem)
210 return NULL;
211
212 hem->refcount = 0;
213 INIT_LIST_HEAD(&hem->chunk_list);
214
215 order = get_order(hem_alloc_size);
216
217 while (npages > 0) {
218 if (!chunk) {
219 chunk = kmalloc(sizeof(*chunk),
220 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
221 if (!chunk)
222 goto fail;
223
224 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
225 chunk->npages = 0;
226 chunk->nsg = 0;
227 memset(chunk->buf, 0, sizeof(chunk->buf));
228 list_add_tail(&chunk->list, &hem->chunk_list);
229 }
230
231 while (1 << order > npages)
232 --order;
233
234
235
236
237
238 mem = &chunk->mem[chunk->npages];
239 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
240 &sg_dma_address(mem), gfp_mask);
241 if (!buf)
242 goto fail;
243
244 chunk->buf[chunk->npages] = buf;
245 sg_dma_len(mem) = PAGE_SIZE << order;
246
247 ++chunk->npages;
248 ++chunk->nsg;
249 npages -= 1 << order;
250 }
251
252 return hem;
253
254fail:
255 hns_roce_free_hem(hr_dev, hem);
256 return NULL;
257}
258
259void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
260{
261 struct hns_roce_hem_chunk *chunk, *tmp;
262 int i;
263
264 if (!hem)
265 return;
266
267 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
268 for (i = 0; i < chunk->npages; ++i)
269 dma_free_coherent(hr_dev->dev,
270 sg_dma_len(&chunk->mem[i]),
271 chunk->buf[i],
272 sg_dma_address(&chunk->mem[i]));
273 kfree(chunk);
274 }
275
276 kfree(hem);
277}
278
279static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
280 struct hns_roce_hem_table *table, unsigned long obj)
281{
282 spinlock_t *lock = &hr_dev->bt_cmd_lock;
283 struct device *dev = hr_dev->dev;
284 unsigned long end = 0;
285 unsigned long flags;
286 struct hns_roce_hem_iter iter;
287 void __iomem *bt_cmd;
288 u32 bt_cmd_h_val = 0;
289 u32 bt_cmd_val[2];
290 u32 bt_cmd_l = 0;
291 u64 bt_ba = 0;
292 int ret = 0;
293
294
295 unsigned long i = (obj & (table->num_obj - 1)) /
296 (table->table_chunk_size / table->obj_size);
297
298 switch (table->type) {
299 case HEM_TYPE_QPC:
300 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
301 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
302 break;
303 case HEM_TYPE_MTPT:
304 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
305 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
306 HEM_TYPE_MTPT);
307 break;
308 case HEM_TYPE_CQC:
309 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
310 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
311 break;
312 case HEM_TYPE_SRQC:
313 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
314 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
315 HEM_TYPE_SRQC);
316 break;
317 default:
318 return ret;
319 }
320 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
321 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
322 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
323 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
324
325
326 for (hns_roce_hem_first(table->hem[i], &iter);
327 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
328 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
329
330 spin_lock_irqsave(lock, flags);
331
332 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
333
334 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
335 while (1) {
336 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
337 if (!(time_before(jiffies, end))) {
338 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
339 spin_unlock_irqrestore(lock, flags);
340 return -EBUSY;
341 }
342 } else {
343 break;
344 }
345 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
346 }
347
348 bt_cmd_l = (u32)bt_ba;
349 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
350 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
351 bt_ba >> BT_BA_SHIFT);
352
353 bt_cmd_val[0] = bt_cmd_l;
354 bt_cmd_val[1] = bt_cmd_h_val;
355 hns_roce_write64_k(bt_cmd_val,
356 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
357 spin_unlock_irqrestore(lock, flags);
358 }
359
360 return ret;
361}
362
363static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
364 struct hns_roce_hem_table *table,
365 unsigned long obj)
366{
367 struct device *dev = hr_dev->dev;
368 struct hns_roce_hem_mhop mhop;
369 struct hns_roce_hem_iter iter;
370 u32 buf_chunk_size;
371 u32 bt_chunk_size;
372 u32 chunk_ba_num;
373 u32 hop_num;
374 u32 size;
375 u32 bt_num;
376 u64 hem_idx;
377 u64 bt_l1_idx = 0;
378 u64 bt_l0_idx = 0;
379 u64 bt_ba;
380 unsigned long mhop_obj = obj;
381 int bt_l1_allocated = 0;
382 int bt_l0_allocated = 0;
383 int step_idx;
384 int ret;
385
386 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
387 if (ret)
388 return ret;
389
390 buf_chunk_size = mhop.buf_chunk_size;
391 bt_chunk_size = mhop.bt_chunk_size;
392 hop_num = mhop.hop_num;
393 chunk_ba_num = bt_chunk_size / 8;
394
395 bt_num = hns_roce_get_bt_num(table->type, hop_num);
396 switch (bt_num) {
397 case 3:
398 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
399 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
400 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
401 bt_l0_idx = mhop.l0_idx;
402 break;
403 case 2:
404 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
405 bt_l0_idx = mhop.l0_idx;
406 break;
407 case 1:
408 hem_idx = mhop.l0_idx;
409 break;
410 default:
411 dev_err(dev, "Table %d not support hop_num = %d!\n",
412 table->type, hop_num);
413 return -EINVAL;
414 }
415
416 mutex_lock(&table->mutex);
417
418 if (table->hem[hem_idx]) {
419 ++table->hem[hem_idx]->refcount;
420 goto out;
421 }
422
423
424 if ((check_whether_bt_num_3(table->type, hop_num) ||
425 check_whether_bt_num_2(table->type, hop_num)) &&
426 !table->bt_l0[bt_l0_idx]) {
427 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
428 &(table->bt_l0_dma_addr[bt_l0_idx]),
429 GFP_KERNEL);
430 if (!table->bt_l0[bt_l0_idx]) {
431 ret = -ENOMEM;
432 goto out;
433 }
434 bt_l0_allocated = 1;
435
436
437 if (table->type < HEM_TYPE_MTT) {
438 step_idx = 0;
439 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
440 ret = -ENODEV;
441 dev_err(dev, "set HEM base address to HW failed!\n");
442 goto err_dma_alloc_l1;
443 }
444 }
445 }
446
447
448 if (check_whether_bt_num_3(table->type, hop_num) &&
449 !table->bt_l1[bt_l1_idx]) {
450 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
451 &(table->bt_l1_dma_addr[bt_l1_idx]),
452 GFP_KERNEL);
453 if (!table->bt_l1[bt_l1_idx]) {
454 ret = -ENOMEM;
455 goto err_dma_alloc_l1;
456 }
457 bt_l1_allocated = 1;
458 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
459 table->bt_l1_dma_addr[bt_l1_idx];
460
461
462 step_idx = 1;
463 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
464 ret = -ENODEV;
465 dev_err(dev, "set HEM base address to HW failed!\n");
466 goto err_alloc_hem_buf;
467 }
468 }
469
470
471
472
473
474 size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
475 table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
476 size >> PAGE_SHIFT,
477 size,
478 (table->lowmem ? GFP_KERNEL :
479 GFP_HIGHUSER) | __GFP_NOWARN);
480 if (!table->hem[hem_idx]) {
481 ret = -ENOMEM;
482 goto err_alloc_hem_buf;
483 }
484
485 hns_roce_hem_first(table->hem[hem_idx], &iter);
486 bt_ba = hns_roce_hem_addr(&iter);
487
488 if (table->type < HEM_TYPE_MTT) {
489 if (hop_num == 2) {
490 *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
491 step_idx = 2;
492 } else if (hop_num == 1) {
493 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
494 step_idx = 1;
495 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
496 step_idx = 0;
497 } else {
498 ret = -EINVAL;
499 goto err_dma_alloc_l1;
500 }
501
502
503 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
504 ret = -ENODEV;
505 dev_err(dev, "set HEM base address to HW failed!\n");
506 goto err_alloc_hem_buf;
507 }
508 } else if (hop_num == 2) {
509 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
510 }
511
512 ++table->hem[hem_idx]->refcount;
513 goto out;
514
515err_alloc_hem_buf:
516 if (bt_l1_allocated) {
517 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
518 table->bt_l1_dma_addr[bt_l1_idx]);
519 table->bt_l1[bt_l1_idx] = NULL;
520 }
521
522err_dma_alloc_l1:
523 if (bt_l0_allocated) {
524 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
525 table->bt_l0_dma_addr[bt_l0_idx]);
526 table->bt_l0[bt_l0_idx] = NULL;
527 }
528
529out:
530 mutex_unlock(&table->mutex);
531 return ret;
532}
533
534int hns_roce_table_get(struct hns_roce_dev *hr_dev,
535 struct hns_roce_hem_table *table, unsigned long obj)
536{
537 struct device *dev = hr_dev->dev;
538 int ret = 0;
539 unsigned long i;
540
541 if (hns_roce_check_whether_mhop(hr_dev, table->type))
542 return hns_roce_table_mhop_get(hr_dev, table, obj);
543
544 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
545 table->obj_size);
546
547 mutex_lock(&table->mutex);
548
549 if (table->hem[i]) {
550 ++table->hem[i]->refcount;
551 goto out;
552 }
553
554 table->hem[i] = hns_roce_alloc_hem(hr_dev,
555 table->table_chunk_size >> PAGE_SHIFT,
556 table->table_chunk_size,
557 (table->lowmem ? GFP_KERNEL :
558 GFP_HIGHUSER) | __GFP_NOWARN);
559 if (!table->hem[i]) {
560 ret = -ENOMEM;
561 goto out;
562 }
563
564
565 if (hns_roce_set_hem(hr_dev, table, obj)) {
566 hns_roce_free_hem(hr_dev, table->hem[i]);
567 table->hem[i] = NULL;
568 ret = -ENODEV;
569 dev_err(dev, "set HEM base address to HW failed.\n");
570 goto out;
571 }
572
573 ++table->hem[i]->refcount;
574out:
575 mutex_unlock(&table->mutex);
576 return ret;
577}
578
579static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
580 struct hns_roce_hem_table *table,
581 unsigned long obj,
582 int check_refcount)
583{
584 struct device *dev = hr_dev->dev;
585 struct hns_roce_hem_mhop mhop;
586 unsigned long mhop_obj = obj;
587 u32 bt_chunk_size;
588 u32 chunk_ba_num;
589 u32 hop_num;
590 u32 start_idx;
591 u32 bt_num;
592 u64 hem_idx;
593 u64 bt_l1_idx = 0;
594 int ret;
595
596 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
597 if (ret)
598 return;
599
600 bt_chunk_size = mhop.bt_chunk_size;
601 hop_num = mhop.hop_num;
602 chunk_ba_num = bt_chunk_size / 8;
603
604 bt_num = hns_roce_get_bt_num(table->type, hop_num);
605 switch (bt_num) {
606 case 3:
607 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
608 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
609 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
610 break;
611 case 2:
612 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
613 break;
614 case 1:
615 hem_idx = mhop.l0_idx;
616 break;
617 default:
618 dev_err(dev, "Table %d not support hop_num = %d!\n",
619 table->type, hop_num);
620 return;
621 }
622
623 mutex_lock(&table->mutex);
624
625 if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
626 mutex_unlock(&table->mutex);
627 return;
628 }
629
630 if (table->type < HEM_TYPE_MTT && hop_num == 1) {
631 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
632 dev_warn(dev, "Clear HEM base address failed.\n");
633 } else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
634 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
635 dev_warn(dev, "Clear HEM base address failed.\n");
636 } else if (table->type < HEM_TYPE_MTT &&
637 hop_num == HNS_ROCE_HOP_NUM_0) {
638 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
639 dev_warn(dev, "Clear HEM base address failed.\n");
640 }
641
642
643
644
645
646 hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
647 table->hem[hem_idx] = NULL;
648
649 if (check_whether_bt_num_2(table->type, hop_num)) {
650 start_idx = mhop.l0_idx * chunk_ba_num;
651 if (hns_roce_check_hem_null(table->hem, start_idx,
652 chunk_ba_num)) {
653 if (table->type < HEM_TYPE_MTT &&
654 hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
655 dev_warn(dev, "Clear HEM base address failed.\n");
656
657 dma_free_coherent(dev, bt_chunk_size,
658 table->bt_l0[mhop.l0_idx],
659 table->bt_l0_dma_addr[mhop.l0_idx]);
660 table->bt_l0[mhop.l0_idx] = NULL;
661 }
662 } else if (check_whether_bt_num_3(table->type, hop_num)) {
663 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
664 mhop.l1_idx * chunk_ba_num;
665 if (hns_roce_check_hem_null(table->hem, start_idx,
666 chunk_ba_num)) {
667 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
668 dev_warn(dev, "Clear HEM base address failed.\n");
669
670 dma_free_coherent(dev, bt_chunk_size,
671 table->bt_l1[bt_l1_idx],
672 table->bt_l1_dma_addr[bt_l1_idx]);
673 table->bt_l1[bt_l1_idx] = NULL;
674
675 start_idx = mhop.l0_idx * chunk_ba_num;
676 if (hns_roce_check_bt_null(table->bt_l1, start_idx,
677 chunk_ba_num)) {
678 if (hr_dev->hw->clear_hem(hr_dev, table, obj,
679 0))
680 dev_warn(dev, "Clear HEM base address failed.\n");
681
682 dma_free_coherent(dev, bt_chunk_size,
683 table->bt_l0[mhop.l0_idx],
684 table->bt_l0_dma_addr[mhop.l0_idx]);
685 table->bt_l0[mhop.l0_idx] = NULL;
686 }
687 }
688 }
689
690 mutex_unlock(&table->mutex);
691}
692
693void hns_roce_table_put(struct hns_roce_dev *hr_dev,
694 struct hns_roce_hem_table *table, unsigned long obj)
695{
696 struct device *dev = hr_dev->dev;
697 unsigned long i;
698
699 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
700 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
701 return;
702 }
703
704 i = (obj & (table->num_obj - 1)) /
705 (table->table_chunk_size / table->obj_size);
706
707 mutex_lock(&table->mutex);
708
709 if (--table->hem[i]->refcount == 0) {
710
711 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
712 dev_warn(dev, "Clear HEM base address failed.\n");
713
714 hns_roce_free_hem(hr_dev, table->hem[i]);
715 table->hem[i] = NULL;
716 }
717
718 mutex_unlock(&table->mutex);
719}
720
721void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
722 struct hns_roce_hem_table *table,
723 unsigned long obj, dma_addr_t *dma_handle)
724{
725 struct hns_roce_hem_chunk *chunk;
726 struct hns_roce_hem_mhop mhop;
727 struct hns_roce_hem *hem;
728 void *addr = NULL;
729 unsigned long mhop_obj = obj;
730 unsigned long obj_per_chunk;
731 unsigned long idx_offset;
732 int offset, dma_offset;
733 int length;
734 int i, j;
735 u32 hem_idx = 0;
736
737 if (!table->lowmem)
738 return NULL;
739
740 mutex_lock(&table->mutex);
741
742 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
743 obj_per_chunk = table->table_chunk_size / table->obj_size;
744 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
745 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
746 dma_offset = offset = idx_offset * table->obj_size;
747 } else {
748 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
749
750 i = mhop.l0_idx;
751 j = mhop.l1_idx;
752 if (mhop.hop_num == 2)
753 hem_idx = i * (mhop.bt_chunk_size / 8) + j;
754 else if (mhop.hop_num == 1 ||
755 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
756 hem_idx = i;
757
758 hem = table->hem[hem_idx];
759 dma_offset = offset = (obj & (table->num_obj - 1)) *
760 table->obj_size % mhop.bt_chunk_size;
761 if (mhop.hop_num == 2)
762 dma_offset = offset = 0;
763 }
764
765 if (!hem)
766 goto out;
767
768 list_for_each_entry(chunk, &hem->chunk_list, list) {
769 for (i = 0; i < chunk->npages; ++i) {
770 length = sg_dma_len(&chunk->mem[i]);
771 if (dma_handle && dma_offset >= 0) {
772 if (length > (u32)dma_offset)
773 *dma_handle = sg_dma_address(
774 &chunk->mem[i]) + dma_offset;
775 dma_offset -= length;
776 }
777
778 if (length > (u32)offset) {
779 addr = chunk->buf[i] + offset;
780 goto out;
781 }
782 offset -= length;
783 }
784 }
785
786out:
787 mutex_unlock(&table->mutex);
788 return addr;
789}
790EXPORT_SYMBOL_GPL(hns_roce_table_find);
791
792int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
793 struct hns_roce_hem_table *table,
794 unsigned long start, unsigned long end)
795{
796 struct hns_roce_hem_mhop mhop;
797 unsigned long inc = table->table_chunk_size / table->obj_size;
798 unsigned long i;
799 int ret;
800
801 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
802 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
803 inc = mhop.bt_chunk_size / table->obj_size;
804 }
805
806
807 for (i = start; i <= end; i += inc) {
808 ret = hns_roce_table_get(hr_dev, table, i);
809 if (ret)
810 goto fail;
811 }
812
813 return 0;
814
815fail:
816 while (i > start) {
817 i -= inc;
818 hns_roce_table_put(hr_dev, table, i);
819 }
820 return ret;
821}
822
823void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
824 struct hns_roce_hem_table *table,
825 unsigned long start, unsigned long end)
826{
827 struct hns_roce_hem_mhop mhop;
828 unsigned long inc = table->table_chunk_size / table->obj_size;
829 unsigned long i;
830
831 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
832 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
833 inc = mhop.bt_chunk_size / table->obj_size;
834 }
835
836 for (i = start; i <= end; i += inc)
837 hns_roce_table_put(hr_dev, table, i);
838}
839
840int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
841 struct hns_roce_hem_table *table, u32 type,
842 unsigned long obj_size, unsigned long nobj,
843 int use_lowmem)
844{
845 struct device *dev = hr_dev->dev;
846 unsigned long obj_per_chunk;
847 unsigned long num_hem;
848
849 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
850 table->table_chunk_size = hr_dev->caps.chunk_sz;
851 obj_per_chunk = table->table_chunk_size / obj_size;
852 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
853
854 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
855 if (!table->hem)
856 return -ENOMEM;
857 } else {
858 unsigned long buf_chunk_size;
859 unsigned long bt_chunk_size;
860 unsigned long bt_chunk_num;
861 unsigned long num_bt_l0 = 0;
862 u32 hop_num;
863
864 switch (type) {
865 case HEM_TYPE_QPC:
866 buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
867 + PAGE_SHIFT);
868 bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
869 + PAGE_SHIFT);
870 num_bt_l0 = hr_dev->caps.qpc_bt_num;
871 hop_num = hr_dev->caps.qpc_hop_num;
872 break;
873 case HEM_TYPE_MTPT:
874 buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
875 + PAGE_SHIFT);
876 bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
877 + PAGE_SHIFT);
878 num_bt_l0 = hr_dev->caps.mpt_bt_num;
879 hop_num = hr_dev->caps.mpt_hop_num;
880 break;
881 case HEM_TYPE_CQC:
882 buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
883 + PAGE_SHIFT);
884 bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
885 + PAGE_SHIFT);
886 num_bt_l0 = hr_dev->caps.cqc_bt_num;
887 hop_num = hr_dev->caps.cqc_hop_num;
888 break;
889 case HEM_TYPE_SRQC:
890 buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
891 + PAGE_SHIFT);
892 bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
893 + PAGE_SHIFT);
894 num_bt_l0 = hr_dev->caps.srqc_bt_num;
895 hop_num = hr_dev->caps.srqc_hop_num;
896 break;
897 case HEM_TYPE_MTT:
898 buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
899 + PAGE_SHIFT);
900 bt_chunk_size = buf_chunk_size;
901 hop_num = hr_dev->caps.mtt_hop_num;
902 break;
903 case HEM_TYPE_CQE:
904 buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
905 + PAGE_SHIFT);
906 bt_chunk_size = buf_chunk_size;
907 hop_num = hr_dev->caps.cqe_hop_num;
908 break;
909 default:
910 dev_err(dev,
911 "Table %d not support to init hem table here!\n",
912 type);
913 return -EINVAL;
914 }
915 obj_per_chunk = buf_chunk_size / obj_size;
916 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
917 bt_chunk_num = bt_chunk_size / 8;
918 if (type >= HEM_TYPE_MTT)
919 num_bt_l0 = bt_chunk_num;
920
921 table->hem = kcalloc(num_hem, sizeof(*table->hem),
922 GFP_KERNEL);
923 if (!table->hem)
924 goto err_kcalloc_hem_buf;
925
926 if (check_whether_bt_num_3(type, hop_num)) {
927 unsigned long num_bt_l1;
928
929 num_bt_l1 = (num_hem + bt_chunk_num - 1) /
930 bt_chunk_num;
931 table->bt_l1 = kcalloc(num_bt_l1,
932 sizeof(*table->bt_l1),
933 GFP_KERNEL);
934 if (!table->bt_l1)
935 goto err_kcalloc_bt_l1;
936
937 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
938 sizeof(*table->bt_l1_dma_addr),
939 GFP_KERNEL);
940
941 if (!table->bt_l1_dma_addr)
942 goto err_kcalloc_l1_dma;
943 }
944
945 if (check_whether_bt_num_2(type, hop_num) ||
946 check_whether_bt_num_3(type, hop_num)) {
947 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
948 GFP_KERNEL);
949 if (!table->bt_l0)
950 goto err_kcalloc_bt_l0;
951
952 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
953 sizeof(*table->bt_l0_dma_addr),
954 GFP_KERNEL);
955 if (!table->bt_l0_dma_addr)
956 goto err_kcalloc_l0_dma;
957 }
958 }
959
960 table->type = type;
961 table->num_hem = num_hem;
962 table->num_obj = nobj;
963 table->obj_size = obj_size;
964 table->lowmem = use_lowmem;
965 mutex_init(&table->mutex);
966
967 return 0;
968
969err_kcalloc_l0_dma:
970 kfree(table->bt_l0);
971 table->bt_l0 = NULL;
972
973err_kcalloc_bt_l0:
974 kfree(table->bt_l1_dma_addr);
975 table->bt_l1_dma_addr = NULL;
976
977err_kcalloc_l1_dma:
978 kfree(table->bt_l1);
979 table->bt_l1 = NULL;
980
981err_kcalloc_bt_l1:
982 kfree(table->hem);
983 table->hem = NULL;
984
985err_kcalloc_hem_buf:
986 return -ENOMEM;
987}
988
989static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
990 struct hns_roce_hem_table *table)
991{
992 struct hns_roce_hem_mhop mhop;
993 u32 buf_chunk_size;
994 int i;
995 u64 obj;
996
997 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
998 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
999 mhop.bt_chunk_size;
1000
1001 for (i = 0; i < table->num_hem; ++i) {
1002 obj = i * buf_chunk_size / table->obj_size;
1003 if (table->hem[i])
1004 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1005 }
1006
1007 kfree(table->hem);
1008 table->hem = NULL;
1009 kfree(table->bt_l1);
1010 table->bt_l1 = NULL;
1011 kfree(table->bt_l1_dma_addr);
1012 table->bt_l1_dma_addr = NULL;
1013 kfree(table->bt_l0);
1014 table->bt_l0 = NULL;
1015 kfree(table->bt_l0_dma_addr);
1016 table->bt_l0_dma_addr = NULL;
1017}
1018
1019void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1020 struct hns_roce_hem_table *table)
1021{
1022 struct device *dev = hr_dev->dev;
1023 unsigned long i;
1024
1025 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1026 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1027 return;
1028 }
1029
1030 for (i = 0; i < table->num_hem; ++i)
1031 if (table->hem[i]) {
1032 if (hr_dev->hw->clear_hem(hr_dev, table,
1033 i * table->table_chunk_size / table->obj_size, 0))
1034 dev_err(dev, "Clear HEM base address failed.\n");
1035
1036 hns_roce_free_hem(hr_dev, table->hem[i]);
1037 }
1038
1039 kfree(table->hem);
1040}
1041
1042void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1043{
1044 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1045 if (hr_dev->caps.trrl_entry_sz)
1046 hns_roce_cleanup_hem_table(hr_dev,
1047 &hr_dev->qp_table.trrl_table);
1048 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1049 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1050 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1051 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1052 hns_roce_cleanup_hem_table(hr_dev,
1053 &hr_dev->mr_table.mtt_cqe_table);
1054 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1055}
1056
