1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/platform_device.h>
35#include "hns_roce_device.h"
36#include "hns_roce_hem.h"
37#include "hns_roce_common.h"
38
39#define DMA_ADDR_T_SHIFT 12
40#define BT_BA_SHIFT 32
41
42bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
43{
44 if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
45 (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
46 (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
47 (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
48 (hr_dev->caps.sccc_hop_num && type == HEM_TYPE_SCCC) ||
49 (hr_dev->caps.qpc_timer_hop_num && type == HEM_TYPE_QPC_TIMER) ||
50 (hr_dev->caps.cqc_timer_hop_num && type == HEM_TYPE_CQC_TIMER) ||
51 (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
52 (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT) ||
53 (hr_dev->caps.srqwqe_hop_num && type == HEM_TYPE_SRQWQE) ||
54 (hr_dev->caps.idx_hop_num && type == HEM_TYPE_IDX))
55 return true;
56
57 return false;
58}
59
60static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
61 u32 bt_chunk_num)
62{
63 int i;
64
65 for (i = 0; i < bt_chunk_num; i++)
66 if (hem[start_idx + i])
67 return false;
68
69 return true;
70}
71
72static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
73{
74 int i;
75
76 for (i = 0; i < bt_chunk_num; i++)
77 if (bt[start_idx + i])
78 return false;
79
80 return true;
81}
82
83static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
84{
85 if (check_whether_bt_num_3(table_type, hop_num))
86 return 3;
87 else if (check_whether_bt_num_2(table_type, hop_num))
88 return 2;
89 else if (check_whether_bt_num_1(table_type, hop_num))
90 return 1;
91 else
92 return 0;
93}
94
95int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
96 struct hns_roce_hem_table *table, unsigned long *obj,
97 struct hns_roce_hem_mhop *mhop)
98{
99 struct device *dev = hr_dev->dev;
100 u32 chunk_ba_num;
101 u32 table_idx;
102 u32 bt_num;
103 u32 chunk_size;
104
105 switch (table->type) {
106 case HEM_TYPE_QPC:
107 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
108 + PAGE_SHIFT);
109 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
110 + PAGE_SHIFT);
111 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
112 mhop->hop_num = hr_dev->caps.qpc_hop_num;
113 break;
114 case HEM_TYPE_MTPT:
115 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
116 + PAGE_SHIFT);
117 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
118 + PAGE_SHIFT);
119 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
120 mhop->hop_num = hr_dev->caps.mpt_hop_num;
121 break;
122 case HEM_TYPE_CQC:
123 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
124 + PAGE_SHIFT);
125 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
126 + PAGE_SHIFT);
127 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
128 mhop->hop_num = hr_dev->caps.cqc_hop_num;
129 break;
130 case HEM_TYPE_SCCC:
131 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
132 + PAGE_SHIFT);
133 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
134 + PAGE_SHIFT);
135 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
136 mhop->hop_num = hr_dev->caps.sccc_hop_num;
137 break;
138 case HEM_TYPE_QPC_TIMER:
139 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
140 + PAGE_SHIFT);
141 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
142 + PAGE_SHIFT);
143 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
144 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
145 break;
146 case HEM_TYPE_CQC_TIMER:
147 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
148 + PAGE_SHIFT);
149 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
150 + PAGE_SHIFT);
151 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
152 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
153 break;
154 case HEM_TYPE_SRQC:
155 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
156 + PAGE_SHIFT);
157 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
158 + PAGE_SHIFT);
159 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
160 mhop->hop_num = hr_dev->caps.srqc_hop_num;
161 break;
162 case HEM_TYPE_MTT:
163 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
164 + PAGE_SHIFT);
165 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
166 + PAGE_SHIFT);
167 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
168 mhop->hop_num = hr_dev->caps.mtt_hop_num;
169 break;
170 case HEM_TYPE_CQE:
171 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
172 + PAGE_SHIFT);
173 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
174 + PAGE_SHIFT);
175 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
176 mhop->hop_num = hr_dev->caps.cqe_hop_num;
177 break;
178 case HEM_TYPE_SRQWQE:
179 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz
180 + PAGE_SHIFT);
181 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
182 + PAGE_SHIFT);
183 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
184 mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
185 break;
186 case HEM_TYPE_IDX:
187 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz
188 + PAGE_SHIFT);
189 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
190 + PAGE_SHIFT);
191 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
192 mhop->hop_num = hr_dev->caps.idx_hop_num;
193 break;
194 default:
195 dev_err(dev, "Table %d not support multi-hop addressing!\n",
196 table->type);
197 return -EINVAL;
198 }
199
200 if (!obj)
201 return 0;
202
203
204
205
206
207 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
208 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
209 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
210 mhop->bt_chunk_size;
211 table_idx = (*obj & (table->num_obj - 1)) /
212 (chunk_size / table->obj_size);
213 switch (bt_num) {
214 case 3:
215 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
216 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
217 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
218 break;
219 case 2:
220 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
221 mhop->l0_idx = table_idx / chunk_ba_num;
222 break;
223 case 1:
224 mhop->l0_idx = table_idx;
225 break;
226 default:
227 dev_err(dev, "Table %d not support hop_num = %d!\n",
228 table->type, mhop->hop_num);
229 return -EINVAL;
230 }
231 if (mhop->l0_idx >= mhop->ba_l0_num)
232 mhop->l0_idx %= mhop->ba_l0_num;
233
234 return 0;
235}
236
237static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
238 int npages,
239 unsigned long hem_alloc_size,
240 gfp_t gfp_mask)
241{
242 struct hns_roce_hem_chunk *chunk = NULL;
243 struct hns_roce_hem *hem;
244 struct scatterlist *mem;
245 int order;
246 void *buf;
247
248 WARN_ON(gfp_mask & __GFP_HIGHMEM);
249
250 hem = kmalloc(sizeof(*hem),
251 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
252 if (!hem)
253 return NULL;
254
255 hem->refcount = 0;
256 INIT_LIST_HEAD(&hem->chunk_list);
257
258 order = get_order(hem_alloc_size);
259
260 while (npages > 0) {
261 if (!chunk) {
262 chunk = kmalloc(sizeof(*chunk),
263 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
264 if (!chunk)
265 goto fail;
266
267 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
268 chunk->npages = 0;
269 chunk->nsg = 0;
270 memset(chunk->buf, 0, sizeof(chunk->buf));
271 list_add_tail(&chunk->list, &hem->chunk_list);
272 }
273
274 while (1 << order > npages)
275 --order;
276
277
278
279
280
281 mem = &chunk->mem[chunk->npages];
282 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
283 &sg_dma_address(mem), gfp_mask);
284 if (!buf)
285 goto fail;
286
287 chunk->buf[chunk->npages] = buf;
288 sg_dma_len(mem) = PAGE_SIZE << order;
289
290 ++chunk->npages;
291 ++chunk->nsg;
292 npages -= 1 << order;
293 }
294
295 return hem;
296
297fail:
298 hns_roce_free_hem(hr_dev, hem);
299 return NULL;
300}
301
302void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
303{
304 struct hns_roce_hem_chunk *chunk, *tmp;
305 int i;
306
307 if (!hem)
308 return;
309
310 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
311 for (i = 0; i < chunk->npages; ++i)
312 dma_free_coherent(hr_dev->dev,
313 sg_dma_len(&chunk->mem[i]),
314 chunk->buf[i],
315 sg_dma_address(&chunk->mem[i]));
316 kfree(chunk);
317 }
318
319 kfree(hem);
320}
321
322static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
323 struct hns_roce_hem_table *table, unsigned long obj)
324{
325 spinlock_t *lock = &hr_dev->bt_cmd_lock;
326 struct device *dev = hr_dev->dev;
327 unsigned long end = 0;
328 unsigned long flags;
329 struct hns_roce_hem_iter iter;
330 void __iomem *bt_cmd;
331 u32 bt_cmd_h_val = 0;
332 u32 bt_cmd_val[2];
333 u32 bt_cmd_l = 0;
334 u64 bt_ba = 0;
335 int ret = 0;
336
337
338 unsigned long i = (obj & (table->num_obj - 1)) /
339 (table->table_chunk_size / table->obj_size);
340
341 switch (table->type) {
342 case HEM_TYPE_QPC:
343 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
344 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
345 break;
346 case HEM_TYPE_MTPT:
347 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
348 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
349 HEM_TYPE_MTPT);
350 break;
351 case HEM_TYPE_CQC:
352 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
353 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
354 break;
355 case HEM_TYPE_SRQC:
356 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
357 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
358 HEM_TYPE_SRQC);
359 break;
360 default:
361 return ret;
362 }
363 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
364 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
365 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
366 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
367
368
369 for (hns_roce_hem_first(table->hem[i], &iter);
370 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
371 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
372
373 spin_lock_irqsave(lock, flags);
374
375 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
376
377 end = HW_SYNC_TIMEOUT_MSECS;
378 while (end) {
379 if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT))
380 break;
381
382 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
383 end -= HW_SYNC_SLEEP_TIME_INTERVAL;
384 }
385
386 if (end <= 0) {
387 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
388 spin_unlock_irqrestore(lock, flags);
389 return -EBUSY;
390 }
391
392 bt_cmd_l = (u32)bt_ba;
393 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
394 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
395 bt_ba >> BT_BA_SHIFT);
396
397 bt_cmd_val[0] = bt_cmd_l;
398 bt_cmd_val[1] = bt_cmd_h_val;
399 hns_roce_write64_k(bt_cmd_val,
400 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
401 spin_unlock_irqrestore(lock, flags);
402 }
403
404 return ret;
405}
406
407static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
408 struct hns_roce_hem_table *table,
409 unsigned long obj)
410{
411 struct device *dev = hr_dev->dev;
412 struct hns_roce_hem_mhop mhop;
413 struct hns_roce_hem_iter iter;
414 u32 buf_chunk_size;
415 u32 bt_chunk_size;
416 u32 chunk_ba_num;
417 u32 hop_num;
418 u32 size;
419 u32 bt_num;
420 u64 hem_idx;
421 u64 bt_l1_idx = 0;
422 u64 bt_l0_idx = 0;
423 u64 bt_ba;
424 unsigned long mhop_obj = obj;
425 int bt_l1_allocated = 0;
426 int bt_l0_allocated = 0;
427 int step_idx;
428 int ret;
429
430 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
431 if (ret)
432 return ret;
433
434 buf_chunk_size = mhop.buf_chunk_size;
435 bt_chunk_size = mhop.bt_chunk_size;
436 hop_num = mhop.hop_num;
437 chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
438
439 bt_num = hns_roce_get_bt_num(table->type, hop_num);
440 switch (bt_num) {
441 case 3:
442 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
443 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
444 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
445 bt_l0_idx = mhop.l0_idx;
446 break;
447 case 2:
448 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
449 bt_l0_idx = mhop.l0_idx;
450 break;
451 case 1:
452 hem_idx = mhop.l0_idx;
453 break;
454 default:
455 dev_err(dev, "Table %d not support hop_num = %d!\n",
456 table->type, hop_num);
457 return -EINVAL;
458 }
459
460 mutex_lock(&table->mutex);
461
462 if (table->hem[hem_idx]) {
463 ++table->hem[hem_idx]->refcount;
464 goto out;
465 }
466
467
468 if ((check_whether_bt_num_3(table->type, hop_num) ||
469 check_whether_bt_num_2(table->type, hop_num)) &&
470 !table->bt_l0[bt_l0_idx]) {
471 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
472 &(table->bt_l0_dma_addr[bt_l0_idx]),
473 GFP_KERNEL);
474 if (!table->bt_l0[bt_l0_idx]) {
475 ret = -ENOMEM;
476 goto out;
477 }
478 bt_l0_allocated = 1;
479
480
481 if (table->type < HEM_TYPE_MTT) {
482 step_idx = 0;
483 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
484 ret = -ENODEV;
485 dev_err(dev, "set HEM base address to HW failed!\n");
486 goto err_dma_alloc_l1;
487 }
488 }
489 }
490
491
492 if (check_whether_bt_num_3(table->type, hop_num) &&
493 !table->bt_l1[bt_l1_idx]) {
494 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
495 &(table->bt_l1_dma_addr[bt_l1_idx]),
496 GFP_KERNEL);
497 if (!table->bt_l1[bt_l1_idx]) {
498 ret = -ENOMEM;
499 goto err_dma_alloc_l1;
500 }
501 bt_l1_allocated = 1;
502 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
503 table->bt_l1_dma_addr[bt_l1_idx];
504
505
506 step_idx = 1;
507 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
508 ret = -ENODEV;
509 dev_err(dev, "set HEM base address to HW failed!\n");
510 goto err_alloc_hem_buf;
511 }
512 }
513
514
515
516
517
518 size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
519 table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
520 size >> PAGE_SHIFT,
521 size,
522 (table->lowmem ? GFP_KERNEL :
523 GFP_HIGHUSER) | __GFP_NOWARN);
524 if (!table->hem[hem_idx]) {
525 ret = -ENOMEM;
526 goto err_alloc_hem_buf;
527 }
528
529 hns_roce_hem_first(table->hem[hem_idx], &iter);
530 bt_ba = hns_roce_hem_addr(&iter);
531
532 if (table->type < HEM_TYPE_MTT) {
533 if (hop_num == 2) {
534 *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
535 step_idx = 2;
536 } else if (hop_num == 1) {
537 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
538 step_idx = 1;
539 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
540 step_idx = 0;
541 } else {
542 ret = -EINVAL;
543 goto err_dma_alloc_l1;
544 }
545
546
547 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
548 ret = -ENODEV;
549 dev_err(dev, "set HEM base address to HW failed!\n");
550 goto err_alloc_hem_buf;
551 }
552 } else if (hop_num == 2) {
553 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
554 }
555
556 ++table->hem[hem_idx]->refcount;
557 goto out;
558
559err_alloc_hem_buf:
560 if (bt_l1_allocated) {
561 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
562 table->bt_l1_dma_addr[bt_l1_idx]);
563 table->bt_l1[bt_l1_idx] = NULL;
564 }
565
566err_dma_alloc_l1:
567 if (bt_l0_allocated) {
568 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
569 table->bt_l0_dma_addr[bt_l0_idx]);
570 table->bt_l0[bt_l0_idx] = NULL;
571 }
572
573out:
574 mutex_unlock(&table->mutex);
575 return ret;
576}
577
578int hns_roce_table_get(struct hns_roce_dev *hr_dev,
579 struct hns_roce_hem_table *table, unsigned long obj)
580{
581 struct device *dev = hr_dev->dev;
582 int ret = 0;
583 unsigned long i;
584
585 if (hns_roce_check_whether_mhop(hr_dev, table->type))
586 return hns_roce_table_mhop_get(hr_dev, table, obj);
587
588 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
589 table->obj_size);
590
591 mutex_lock(&table->mutex);
592
593 if (table->hem[i]) {
594 ++table->hem[i]->refcount;
595 goto out;
596 }
597
598 table->hem[i] = hns_roce_alloc_hem(hr_dev,
599 table->table_chunk_size >> PAGE_SHIFT,
600 table->table_chunk_size,
601 (table->lowmem ? GFP_KERNEL :
602 GFP_HIGHUSER) | __GFP_NOWARN);
603 if (!table->hem[i]) {
604 ret = -ENOMEM;
605 goto out;
606 }
607
608
609 if (hns_roce_set_hem(hr_dev, table, obj)) {
610 hns_roce_free_hem(hr_dev, table->hem[i]);
611 table->hem[i] = NULL;
612 ret = -ENODEV;
613 dev_err(dev, "set HEM base address to HW failed.\n");
614 goto out;
615 }
616
617 ++table->hem[i]->refcount;
618out:
619 mutex_unlock(&table->mutex);
620 return ret;
621}
622
623static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
624 struct hns_roce_hem_table *table,
625 unsigned long obj,
626 int check_refcount)
627{
628 struct device *dev = hr_dev->dev;
629 struct hns_roce_hem_mhop mhop;
630 unsigned long mhop_obj = obj;
631 u32 bt_chunk_size;
632 u32 chunk_ba_num;
633 u32 hop_num;
634 u32 start_idx;
635 u32 bt_num;
636 u64 hem_idx;
637 u64 bt_l1_idx = 0;
638 int ret;
639
640 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
641 if (ret)
642 return;
643
644 bt_chunk_size = mhop.bt_chunk_size;
645 hop_num = mhop.hop_num;
646 chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
647
648 bt_num = hns_roce_get_bt_num(table->type, hop_num);
649 switch (bt_num) {
650 case 3:
651 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
652 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
653 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
654 break;
655 case 2:
656 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
657 break;
658 case 1:
659 hem_idx = mhop.l0_idx;
660 break;
661 default:
662 dev_err(dev, "Table %d not support hop_num = %d!\n",
663 table->type, hop_num);
664 return;
665 }
666
667 mutex_lock(&table->mutex);
668
669 if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
670 mutex_unlock(&table->mutex);
671 return;
672 }
673
674 if (table->type < HEM_TYPE_MTT && hop_num == 1) {
675 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
676 dev_warn(dev, "Clear HEM base address failed.\n");
677 } else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
678 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
679 dev_warn(dev, "Clear HEM base address failed.\n");
680 } else if (table->type < HEM_TYPE_MTT &&
681 hop_num == HNS_ROCE_HOP_NUM_0) {
682 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
683 dev_warn(dev, "Clear HEM base address failed.\n");
684 }
685
686
687
688
689
690 hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
691 table->hem[hem_idx] = NULL;
692
693 if (check_whether_bt_num_2(table->type, hop_num)) {
694 start_idx = mhop.l0_idx * chunk_ba_num;
695 if (hns_roce_check_hem_null(table->hem, start_idx,
696 chunk_ba_num)) {
697 if (table->type < HEM_TYPE_MTT &&
698 hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
699 dev_warn(dev, "Clear HEM base address failed.\n");
700
701 dma_free_coherent(dev, bt_chunk_size,
702 table->bt_l0[mhop.l0_idx],
703 table->bt_l0_dma_addr[mhop.l0_idx]);
704 table->bt_l0[mhop.l0_idx] = NULL;
705 }
706 } else if (check_whether_bt_num_3(table->type, hop_num)) {
707 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
708 mhop.l1_idx * chunk_ba_num;
709 if (hns_roce_check_hem_null(table->hem, start_idx,
710 chunk_ba_num)) {
711 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
712 dev_warn(dev, "Clear HEM base address failed.\n");
713
714 dma_free_coherent(dev, bt_chunk_size,
715 table->bt_l1[bt_l1_idx],
716 table->bt_l1_dma_addr[bt_l1_idx]);
717 table->bt_l1[bt_l1_idx] = NULL;
718
719 start_idx = mhop.l0_idx * chunk_ba_num;
720 if (hns_roce_check_bt_null(table->bt_l1, start_idx,
721 chunk_ba_num)) {
722 if (hr_dev->hw->clear_hem(hr_dev, table, obj,
723 0))
724 dev_warn(dev, "Clear HEM base address failed.\n");
725
726 dma_free_coherent(dev, bt_chunk_size,
727 table->bt_l0[mhop.l0_idx],
728 table->bt_l0_dma_addr[mhop.l0_idx]);
729 table->bt_l0[mhop.l0_idx] = NULL;
730 }
731 }
732 }
733
734 mutex_unlock(&table->mutex);
735}
736
737void hns_roce_table_put(struct hns_roce_dev *hr_dev,
738 struct hns_roce_hem_table *table, unsigned long obj)
739{
740 struct device *dev = hr_dev->dev;
741 unsigned long i;
742
743 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
744 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
745 return;
746 }
747
748 i = (obj & (table->num_obj - 1)) /
749 (table->table_chunk_size / table->obj_size);
750
751 mutex_lock(&table->mutex);
752
753 if (--table->hem[i]->refcount == 0) {
754
755 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
756 dev_warn(dev, "Clear HEM base address failed.\n");
757
758 hns_roce_free_hem(hr_dev, table->hem[i]);
759 table->hem[i] = NULL;
760 }
761
762 mutex_unlock(&table->mutex);
763}
764
765void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
766 struct hns_roce_hem_table *table,
767 unsigned long obj, dma_addr_t *dma_handle)
768{
769 struct hns_roce_hem_chunk *chunk;
770 struct hns_roce_hem_mhop mhop;
771 struct hns_roce_hem *hem;
772 void *addr = NULL;
773 unsigned long mhop_obj = obj;
774 unsigned long obj_per_chunk;
775 unsigned long idx_offset;
776 int offset, dma_offset;
777 int length;
778 int i, j;
779 u32 hem_idx = 0;
780
781 if (!table->lowmem)
782 return NULL;
783
784 mutex_lock(&table->mutex);
785
786 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
787 obj_per_chunk = table->table_chunk_size / table->obj_size;
788 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
789 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
790 dma_offset = offset = idx_offset * table->obj_size;
791 } else {
792 u32 seg_size = 64;
793
794 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
795
796 i = mhop.l0_idx;
797 j = mhop.l1_idx;
798 if (mhop.hop_num == 2)
799 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
800 else if (mhop.hop_num == 1 ||
801 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
802 hem_idx = i;
803
804 hem = table->hem[hem_idx];
805 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %
806 mhop.bt_chunk_size;
807 if (mhop.hop_num == 2)
808 dma_offset = offset = 0;
809 }
810
811 if (!hem)
812 goto out;
813
814 list_for_each_entry(chunk, &hem->chunk_list, list) {
815 for (i = 0; i < chunk->npages; ++i) {
816 length = sg_dma_len(&chunk->mem[i]);
817 if (dma_handle && dma_offset >= 0) {
818 if (length > (u32)dma_offset)
819 *dma_handle = sg_dma_address(
820 &chunk->mem[i]) + dma_offset;
821 dma_offset -= length;
822 }
823
824 if (length > (u32)offset) {
825 addr = chunk->buf[i] + offset;
826 goto out;
827 }
828 offset -= length;
829 }
830 }
831
832out:
833 mutex_unlock(&table->mutex);
834 return addr;
835}
836
837int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
838 struct hns_roce_hem_table *table,
839 unsigned long start, unsigned long end)
840{
841 struct hns_roce_hem_mhop mhop;
842 unsigned long inc = table->table_chunk_size / table->obj_size;
843 unsigned long i;
844 int ret;
845
846 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
847 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
848 inc = mhop.bt_chunk_size / table->obj_size;
849 }
850
851
852 for (i = start; i <= end; i += inc) {
853 ret = hns_roce_table_get(hr_dev, table, i);
854 if (ret)
855 goto fail;
856 }
857
858 return 0;
859
860fail:
861 while (i > start) {
862 i -= inc;
863 hns_roce_table_put(hr_dev, table, i);
864 }
865 return ret;
866}
867
868void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
869 struct hns_roce_hem_table *table,
870 unsigned long start, unsigned long end)
871{
872 struct hns_roce_hem_mhop mhop;
873 unsigned long inc = table->table_chunk_size / table->obj_size;
874 unsigned long i;
875
876 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
877 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
878 inc = mhop.bt_chunk_size / table->obj_size;
879 }
880
881 for (i = start; i <= end; i += inc)
882 hns_roce_table_put(hr_dev, table, i);
883}
884
885int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
886 struct hns_roce_hem_table *table, u32 type,
887 unsigned long obj_size, unsigned long nobj,
888 int use_lowmem)
889{
890 struct device *dev = hr_dev->dev;
891 unsigned long obj_per_chunk;
892 unsigned long num_hem;
893
894 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
895 table->table_chunk_size = hr_dev->caps.chunk_sz;
896 obj_per_chunk = table->table_chunk_size / obj_size;
897 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
898
899 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
900 if (!table->hem)
901 return -ENOMEM;
902 } else {
903 unsigned long buf_chunk_size;
904 unsigned long bt_chunk_size;
905 unsigned long bt_chunk_num;
906 unsigned long num_bt_l0 = 0;
907 u32 hop_num;
908
909 switch (type) {
910 case HEM_TYPE_QPC:
911 buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
912 + PAGE_SHIFT);
913 bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
914 + PAGE_SHIFT);
915 num_bt_l0 = hr_dev->caps.qpc_bt_num;
916 hop_num = hr_dev->caps.qpc_hop_num;
917 break;
918 case HEM_TYPE_MTPT:
919 buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
920 + PAGE_SHIFT);
921 bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
922 + PAGE_SHIFT);
923 num_bt_l0 = hr_dev->caps.mpt_bt_num;
924 hop_num = hr_dev->caps.mpt_hop_num;
925 break;
926 case HEM_TYPE_CQC:
927 buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
928 + PAGE_SHIFT);
929 bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
930 + PAGE_SHIFT);
931 num_bt_l0 = hr_dev->caps.cqc_bt_num;
932 hop_num = hr_dev->caps.cqc_hop_num;
933 break;
934 case HEM_TYPE_SCCC:
935 buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
936 + PAGE_SHIFT);
937 bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
938 + PAGE_SHIFT);
939 num_bt_l0 = hr_dev->caps.sccc_bt_num;
940 hop_num = hr_dev->caps.sccc_hop_num;
941 break;
942 case HEM_TYPE_QPC_TIMER:
943 buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
944 + PAGE_SHIFT);
945 bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
946 + PAGE_SHIFT);
947 num_bt_l0 = hr_dev->caps.qpc_timer_bt_num;
948 hop_num = hr_dev->caps.qpc_timer_hop_num;
949 break;
950 case HEM_TYPE_CQC_TIMER:
951 buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
952 + PAGE_SHIFT);
953 bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
954 + PAGE_SHIFT);
955 num_bt_l0 = hr_dev->caps.cqc_timer_bt_num;
956 hop_num = hr_dev->caps.cqc_timer_hop_num;
957 break;
958 case HEM_TYPE_SRQC:
959 buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
960 + PAGE_SHIFT);
961 bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
962 + PAGE_SHIFT);
963 num_bt_l0 = hr_dev->caps.srqc_bt_num;
964 hop_num = hr_dev->caps.srqc_hop_num;
965 break;
966 case HEM_TYPE_MTT:
967 buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
968 + PAGE_SHIFT);
969 bt_chunk_size = buf_chunk_size;
970 hop_num = hr_dev->caps.mtt_hop_num;
971 break;
972 case HEM_TYPE_CQE:
973 buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
974 + PAGE_SHIFT);
975 bt_chunk_size = buf_chunk_size;
976 hop_num = hr_dev->caps.cqe_hop_num;
977 break;
978 case HEM_TYPE_SRQWQE:
979 buf_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
980 + PAGE_SHIFT);
981 bt_chunk_size = buf_chunk_size;
982 hop_num = hr_dev->caps.srqwqe_hop_num;
983 break;
984 case HEM_TYPE_IDX:
985 buf_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
986 + PAGE_SHIFT);
987 bt_chunk_size = buf_chunk_size;
988 hop_num = hr_dev->caps.idx_hop_num;
989 break;
990 default:
991 dev_err(dev,
992 "Table %d not support to init hem table here!\n",
993 type);
994 return -EINVAL;
995 }
996 obj_per_chunk = buf_chunk_size / obj_size;
997 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
998 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
999 if (type >= HEM_TYPE_MTT)
1000 num_bt_l0 = bt_chunk_num;
1001
1002 table->hem = kcalloc(num_hem, sizeof(*table->hem),
1003 GFP_KERNEL);
1004 if (!table->hem)
1005 goto err_kcalloc_hem_buf;
1006
1007 if (check_whether_bt_num_3(type, hop_num)) {
1008 unsigned long num_bt_l1;
1009
1010 num_bt_l1 = (num_hem + bt_chunk_num - 1) /
1011 bt_chunk_num;
1012 table->bt_l1 = kcalloc(num_bt_l1,
1013 sizeof(*table->bt_l1),
1014 GFP_KERNEL);
1015 if (!table->bt_l1)
1016 goto err_kcalloc_bt_l1;
1017
1018 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
1019 sizeof(*table->bt_l1_dma_addr),
1020 GFP_KERNEL);
1021
1022 if (!table->bt_l1_dma_addr)
1023 goto err_kcalloc_l1_dma;
1024 }
1025
1026 if (check_whether_bt_num_2(type, hop_num) ||
1027 check_whether_bt_num_3(type, hop_num)) {
1028 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
1029 GFP_KERNEL);
1030 if (!table->bt_l0)
1031 goto err_kcalloc_bt_l0;
1032
1033 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
1034 sizeof(*table->bt_l0_dma_addr),
1035 GFP_KERNEL);
1036 if (!table->bt_l0_dma_addr)
1037 goto err_kcalloc_l0_dma;
1038 }
1039 }
1040
1041 table->type = type;
1042 table->num_hem = num_hem;
1043 table->num_obj = nobj;
1044 table->obj_size = obj_size;
1045 table->lowmem = use_lowmem;
1046 mutex_init(&table->mutex);
1047
1048 return 0;
1049
1050err_kcalloc_l0_dma:
1051 kfree(table->bt_l0);
1052 table->bt_l0 = NULL;
1053
1054err_kcalloc_bt_l0:
1055 kfree(table->bt_l1_dma_addr);
1056 table->bt_l1_dma_addr = NULL;
1057
1058err_kcalloc_l1_dma:
1059 kfree(table->bt_l1);
1060 table->bt_l1 = NULL;
1061
1062err_kcalloc_bt_l1:
1063 kfree(table->hem);
1064 table->hem = NULL;
1065
1066err_kcalloc_hem_buf:
1067 return -ENOMEM;
1068}
1069
1070static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
1071 struct hns_roce_hem_table *table)
1072{
1073 struct hns_roce_hem_mhop mhop;
1074 u32 buf_chunk_size;
1075 int i;
1076 u64 obj;
1077
1078 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
1079 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1080 mhop.bt_chunk_size;
1081
1082 for (i = 0; i < table->num_hem; ++i) {
1083 obj = i * buf_chunk_size / table->obj_size;
1084 if (table->hem[i])
1085 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1086 }
1087
1088 kfree(table->hem);
1089 table->hem = NULL;
1090 kfree(table->bt_l1);
1091 table->bt_l1 = NULL;
1092 kfree(table->bt_l1_dma_addr);
1093 table->bt_l1_dma_addr = NULL;
1094 kfree(table->bt_l0);
1095 table->bt_l0 = NULL;
1096 kfree(table->bt_l0_dma_addr);
1097 table->bt_l0_dma_addr = NULL;
1098}
1099
1100void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1101 struct hns_roce_hem_table *table)
1102{
1103 struct device *dev = hr_dev->dev;
1104 unsigned long i;
1105
1106 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1107 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1108 return;
1109 }
1110
1111 for (i = 0; i < table->num_hem; ++i)
1112 if (table->hem[i]) {
1113 if (hr_dev->hw->clear_hem(hr_dev, table,
1114 i * table->table_chunk_size / table->obj_size, 0))
1115 dev_err(dev, "Clear HEM base address failed.\n");
1116
1117 hns_roce_free_hem(hr_dev, table->hem[i]);
1118 }
1119
1120 kfree(table->hem);
1121}
1122
1123void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1124{
1125 if ((hr_dev->caps.num_idx_segs))
1126 hns_roce_cleanup_hem_table(hr_dev,
1127 &hr_dev->mr_table.mtt_idx_table);
1128 if (hr_dev->caps.num_srqwqe_segs)
1129 hns_roce_cleanup_hem_table(hr_dev,
1130 &hr_dev->mr_table.mtt_srqwqe_table);
1131 if (hr_dev->caps.srqc_entry_sz)
1132 hns_roce_cleanup_hem_table(hr_dev,
1133 &hr_dev->srq_table.table);
1134 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1135 if (hr_dev->caps.qpc_timer_entry_sz)
1136 hns_roce_cleanup_hem_table(hr_dev,
1137 &hr_dev->qpc_timer_table);
1138 if (hr_dev->caps.cqc_timer_entry_sz)
1139 hns_roce_cleanup_hem_table(hr_dev,
1140 &hr_dev->cqc_timer_table);
1141 if (hr_dev->caps.sccc_entry_sz)
1142 hns_roce_cleanup_hem_table(hr_dev,
1143 &hr_dev->qp_table.sccc_table);
1144 if (hr_dev->caps.trrl_entry_sz)
1145 hns_roce_cleanup_hem_table(hr_dev,
1146 &hr_dev->qp_table.trrl_table);
1147 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1148 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1149 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1150 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1151 hns_roce_cleanup_hem_table(hr_dev,
1152 &hr_dev->mr_table.mtt_cqe_table);
1153 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1154}
1155
1156struct roce_hem_item {
1157 struct list_head list;
1158 struct list_head sibling;
1159 void *addr;
1160 dma_addr_t dma_addr;
1161 size_t count;
1162 int start;
1163 int end;
1164};
1165
1166static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev,
1167 int start, int end,
1168 int count, bool exist_bt,
1169 int bt_level)
1170{
1171 struct roce_hem_item *hem;
1172
1173 hem = kzalloc(sizeof(*hem), GFP_KERNEL);
1174 if (!hem)
1175 return NULL;
1176
1177 if (exist_bt) {
1178 hem->addr = dma_alloc_coherent(hr_dev->dev,
1179 count * BA_BYTE_LEN,
1180 &hem->dma_addr, GFP_KERNEL);
1181 if (!hem->addr) {
1182 kfree(hem);
1183 return NULL;
1184 }
1185 }
1186
1187 hem->count = count;
1188 hem->start = start;
1189 hem->end = end;
1190 INIT_LIST_HEAD(&hem->list);
1191 INIT_LIST_HEAD(&hem->sibling);
1192
1193 return hem;
1194}
1195
1196static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1197 struct roce_hem_item *hem, bool exist_bt)
1198{
1199 if (exist_bt)
1200 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1201 hem->addr, hem->dma_addr);
1202 kfree(hem);
1203}
1204
1205static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1206 struct list_head *head, bool exist_bt)
1207{
1208 struct roce_hem_item *hem, *temp_hem;
1209
1210 list_for_each_entry_safe(hem, temp_hem, head, list) {
1211 list_del(&hem->list);
1212 hem_list_free_item(hr_dev, hem, exist_bt);
1213 }
1214}
1215
1216static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1217 u64 table_addr)
1218{
1219 *(u64 *)(base_addr) = table_addr;
1220}
1221
1222
1223static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1224 struct roce_hem_item *hem, void *cpu_addr,
1225 u64 phy_addr)
1226{
1227 hem->addr = cpu_addr;
1228 hem->dma_addr = (dma_addr_t)phy_addr;
1229}
1230
1231static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem,
1232 int offset)
1233{
1234 return (hem->start <= offset && offset <= hem->end);
1235}
1236
1237static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1238 int page_offset)
1239{
1240 struct roce_hem_item *hem, *temp_hem;
1241 struct roce_hem_item *found = NULL;
1242
1243 list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1244 if (hem_list_page_is_in_range(hem, page_offset)) {
1245 found = hem;
1246 break;
1247 }
1248 }
1249
1250 return found;
1251}
1252
1253static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1254{
1255
1256
1257
1258
1259
1260
1261
1262 return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1263}
1264
1265
1266
1267
1268
1269
1270
1271static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1272{
1273 u32 step;
1274 int max;
1275 int i;
1276
1277 if (hopnum <= bt_level)
1278 return 0;
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290 step = 1;
1291 max = hopnum - bt_level;
1292 for (i = 0; i < max; i++)
1293 step = step * unit;
1294
1295 return step;
1296}
1297
1298
1299
1300
1301
1302
1303
1304int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1305 int region_cnt, int unit)
1306{
1307 struct hns_roce_buf_region *r;
1308 int total = 0;
1309 int step;
1310 int i;
1311
1312 for (i = 0; i < region_cnt; i++) {
1313 r = (struct hns_roce_buf_region *)®ions[i];
1314 if (r->hopnum > 1) {
1315 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1316 if (step > 0)
1317 total += (r->count + step - 1) / step;
1318 } else {
1319 total += r->count;
1320 }
1321 }
1322
1323 return total;
1324}
1325
1326static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1327 const struct hns_roce_buf_region *r, int unit,
1328 int offset, struct list_head *mid_bt,
1329 struct list_head *btm_bt)
1330{
1331 struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1332 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1333 struct roce_hem_item *cur, *pre;
1334 const int hopnum = r->hopnum;
1335 int start_aligned;
1336 int distance;
1337 int ret = 0;
1338 int max_ofs;
1339 int level;
1340 u32 step;
1341 int end;
1342
1343 if (hopnum <= 1)
1344 return 0;
1345
1346 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1347 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1348 return -EINVAL;
1349 }
1350
1351 if (offset < r->offset) {
1352 dev_err(hr_dev->dev, "invalid offset %d,min %d!\n",
1353 offset, r->offset);
1354 return -EINVAL;
1355 }
1356
1357 distance = offset - r->offset;
1358 max_ofs = r->offset + r->count - 1;
1359 for (level = 0; level < hopnum; level++)
1360 INIT_LIST_HEAD(&temp_list[level]);
1361
1362
1363 for (level = 1; level < hopnum; level++) {
1364 cur = hem_list_search_item(&mid_bt[level], offset);
1365 if (cur) {
1366 hem_ptrs[level] = cur;
1367 continue;
1368 }
1369
1370 step = hem_list_calc_ba_range(hopnum, level, unit);
1371 if (step < 1) {
1372 ret = -EINVAL;
1373 goto err_exit;
1374 }
1375
1376 start_aligned = (distance / step) * step + r->offset;
1377 end = min_t(int, start_aligned + step - 1, max_ofs);
1378 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1379 true, level);
1380 if (!cur) {
1381 ret = -ENOMEM;
1382 goto err_exit;
1383 }
1384 hem_ptrs[level] = cur;
1385 list_add(&cur->list, &temp_list[level]);
1386 if (hem_list_is_bottom_bt(hopnum, level))
1387 list_add(&cur->sibling, &temp_list[0]);
1388
1389
1390 if (level > 1) {
1391 pre = hem_ptrs[level - 1];
1392 step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1393 hem_list_link_bt(hr_dev, pre->addr + step,
1394 cur->dma_addr);
1395 }
1396 }
1397
1398 list_splice(&temp_list[0], btm_bt);
1399 for (level = 1; level < hopnum; level++)
1400 list_splice(&temp_list[level], &mid_bt[level]);
1401
1402 return 0;
1403
1404err_exit:
1405 for (level = 1; level < hopnum; level++)
1406 hem_list_free_all(hr_dev, &temp_list[level], true);
1407
1408 return ret;
1409}
1410
1411static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1412 struct hns_roce_hem_list *hem_list, int unit,
1413 const struct hns_roce_buf_region *regions,
1414 int region_cnt)
1415{
1416 struct roce_hem_item *hem, *temp_hem, *root_hem;
1417 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
1418 const struct hns_roce_buf_region *r;
1419 struct list_head temp_root;
1420 struct list_head temp_btm;
1421 void *cpu_base;
1422 u64 phy_base;
1423 int ret = 0;
1424 int offset;
1425 int total;
1426 int step;
1427 int i;
1428
1429 r = ®ions[0];
1430 root_hem = hem_list_search_item(&hem_list->root_bt, r->offset);
1431 if (root_hem)
1432 return 0;
1433
1434 INIT_LIST_HEAD(&temp_root);
1435 total = r->offset;
1436
1437 r = ®ions[region_cnt - 1];
1438 root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1,
1439 unit, true, 0);
1440 if (!root_hem)
1441 return -ENOMEM;
1442 list_add(&root_hem->list, &temp_root);
1443
1444 hem_list->root_ba = root_hem->dma_addr;
1445
1446 INIT_LIST_HEAD(&temp_btm);
1447 for (i = 0; i < region_cnt; i++)
1448 INIT_LIST_HEAD(&temp_list[i]);
1449
1450 total = 0;
1451 for (i = 0; i < region_cnt && total < unit; i++) {
1452 r = ®ions[i];
1453 if (!r->count)
1454 continue;
1455
1456
1457 cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1458 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1459
1460
1461
1462
1463 if (hem_list_is_bottom_bt(r->hopnum, 0)) {
1464 hem = hem_list_alloc_item(hr_dev, r->offset,
1465 r->offset + r->count - 1,
1466 r->count, false, 0);
1467 if (!hem) {
1468 ret = -ENOMEM;
1469 goto err_exit;
1470 }
1471 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1472 list_add(&hem->list, &temp_list[i]);
1473 list_add(&hem->sibling, &temp_btm);
1474 total += r->count;
1475 } else {
1476 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1477 if (step < 1) {
1478 ret = -EINVAL;
1479 goto err_exit;
1480 }
1481
1482 list_for_each_entry_safe(hem, temp_hem,
1483 &hem_list->mid_bt[i][1], list) {
1484 offset = hem->start / step * BA_BYTE_LEN;
1485 hem_list_link_bt(hr_dev, cpu_base + offset,
1486 hem->dma_addr);
1487 total++;
1488 }
1489 }
1490 }
1491
1492 list_splice(&temp_btm, &hem_list->btm_bt);
1493 list_splice(&temp_root, &hem_list->root_bt);
1494 for (i = 0; i < region_cnt; i++)
1495 list_splice(&temp_list[i], &hem_list->mid_bt[i][0]);
1496
1497 return 0;
1498
1499err_exit:
1500 for (i = 0; i < region_cnt; i++)
1501 hem_list_free_all(hr_dev, &temp_list[i], false);
1502
1503 hem_list_free_all(hr_dev, &temp_root, true);
1504
1505 return ret;
1506}
1507
1508
1509int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1510 struct hns_roce_hem_list *hem_list,
1511 const struct hns_roce_buf_region *regions,
1512 int region_cnt)
1513{
1514 const struct hns_roce_buf_region *r;
1515 int ofs, end;
1516 int ret = 0;
1517 int unit;
1518 int i;
1519
1520 if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1521 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1522 region_cnt);
1523 return -EINVAL;
1524 }
1525
1526 unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN;
1527 for (i = 0; i < region_cnt; i++) {
1528 r = ®ions[i];
1529 if (!r->count)
1530 continue;
1531
1532 end = r->offset + r->count;
1533 for (ofs = r->offset; ofs < end; ofs += unit) {
1534 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1535 hem_list->mid_bt[i],
1536 &hem_list->btm_bt);
1537 if (ret) {
1538 dev_err(hr_dev->dev,
1539 "alloc hem trunk fail ret=%d!\n", ret);
1540 goto err_alloc;
1541 }
1542 }
1543 }
1544
1545 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1546 region_cnt);
1547 if (ret)
1548 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1549 else
1550 return 0;
1551
1552err_alloc:
1553 hns_roce_hem_list_release(hr_dev, hem_list);
1554
1555 return ret;
1556}
1557
1558void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1559 struct hns_roce_hem_list *hem_list)
1560{
1561 int i, j;
1562
1563 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1564 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1565 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1566 j != 0);
1567
1568 hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1569 INIT_LIST_HEAD(&hem_list->btm_bt);
1570 hem_list->root_ba = 0;
1571}
1572
1573void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
1574 int bt_page_order)
1575{
1576 int i, j;
1577
1578 INIT_LIST_HEAD(&hem_list->root_bt);
1579 INIT_LIST_HEAD(&hem_list->btm_bt);
1580 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1581 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1582 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1583
1584 hem_list->bt_pg_shift = bt_page_order;
1585}
1586
1587void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1588 struct hns_roce_hem_list *hem_list,
1589 int offset, int *mtt_cnt, u64 *phy_addr)
1590{
1591 struct list_head *head = &hem_list->btm_bt;
1592 struct roce_hem_item *hem, *temp_hem;
1593 void *cpu_base = NULL;
1594 u64 phy_base = 0;
1595 int nr = 0;
1596
1597 list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1598 if (hem_list_page_is_in_range(hem, offset)) {
1599 nr = offset - hem->start;
1600 cpu_base = hem->addr + nr * BA_BYTE_LEN;
1601 phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1602 nr = hem->end + 1 - offset;
1603 break;
1604 }
1605 }
1606
1607 if (mtt_cnt)
1608 *mtt_cnt = nr;
1609
1610 if (phy_addr)
1611 *phy_addr = phy_base;
1612
1613 return cpu_base;
1614}
1615
