1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#include "i40iw_status.h"
36#include "i40iw_osdep.h"
37#include "i40iw_register.h"
38#include "i40iw_hmc.h"
39
40#include "i40iw_d.h"
41#include "i40iw_type.h"
42#include "i40iw_p.h"
43
44#include <linux/pci.h>
45#include <linux/genalloc.h>
46#include <linux/vmalloc.h>
47#include "i40iw_pble.h"
48#include "i40iw.h"
49
50struct i40iw_device;
51static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
52 struct i40iw_hmc_pble_rsrc *pble_rsrc);
53static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk);
54
55
56
57
58
59void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc)
60{
61 struct list_head *clist;
62 struct list_head *tlist;
63 struct i40iw_chunk *chunk;
64 struct i40iw_pble_pool *pinfo = &pble_rsrc->pinfo;
65
66 if (pinfo->pool) {
67 list_for_each_safe(clist, tlist, &pinfo->clist) {
68 chunk = list_entry(clist, struct i40iw_chunk, list);
69 if (chunk->type == I40IW_VMALLOC)
70 i40iw_free_vmalloc_mem(dev->hw, chunk);
71 kfree(chunk);
72 }
73 gen_pool_destroy(pinfo->pool);
74 }
75}
76
77
78
79
80
81
82enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,
83 struct i40iw_hmc_pble_rsrc *pble_rsrc)
84{
85 struct i40iw_hmc_info *hmc_info;
86 u32 fpm_idx = 0;
87
88 hmc_info = dev->hmc_info;
89 pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].base;
90
91 if (pble_rsrc->fpm_base_addr & 0xfff)
92 fpm_idx = (PAGE_SIZE - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
93
94 pble_rsrc->unallocated_pble =
95 hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt - fpm_idx;
96 pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
97
98 pble_rsrc->pinfo.pool_shift = POOL_SHIFT;
99 pble_rsrc->pinfo.pool = gen_pool_create(pble_rsrc->pinfo.pool_shift, -1);
100 INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
101 if (!pble_rsrc->pinfo.pool)
102 goto error;
103
104 if (add_pble_pool(dev, pble_rsrc))
105 goto error;
106
107 return 0;
108
109 error:i40iw_destroy_pble_pool(dev, pble_rsrc);
110 return I40IW_ERR_NO_MEMORY;
111}
112
113
114
115
116
117
118static inline void get_sd_pd_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc,
119 struct sd_pd_idx *idx)
120{
121 idx->sd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_DIRECT_BP_SIZE;
122 idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_PAGED_BP_SIZE;
123 idx->rel_pd_idx = (idx->pd_idx % I40IW_HMC_PD_CNT_IN_SD);
124}
125
126
127
128
129
130
131
132static enum i40iw_status_code add_sd_direct(struct i40iw_sc_dev *dev,
133 struct i40iw_hmc_pble_rsrc *pble_rsrc,
134 struct i40iw_add_page_info *info)
135{
136 enum i40iw_status_code ret_code = 0;
137 struct sd_pd_idx *idx = &info->idx;
138 struct i40iw_chunk *chunk = info->chunk;
139 struct i40iw_hmc_info *hmc_info = info->hmc_info;
140 struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
141 u32 offset = 0;
142
143 if (!sd_entry->valid) {
144 if (dev->is_pf) {
145 ret_code = i40iw_add_sd_table_entry(dev->hw, hmc_info,
146 info->idx.sd_idx,
147 I40IW_SD_TYPE_DIRECT,
148 I40IW_HMC_DIRECT_BP_SIZE);
149 if (ret_code)
150 return ret_code;
151 chunk->type = I40IW_DMA_COHERENT;
152 }
153 }
154 offset = idx->rel_pd_idx << I40IW_HMC_PAGED_BP_SHIFT;
155 chunk->size = info->pages << I40IW_HMC_PAGED_BP_SHIFT;
156 chunk->vaddr = ((u8 *)sd_entry->u.bp.addr.va + offset);
157 chunk->fpm_addr = pble_rsrc->next_fpm_addr;
158 i40iw_debug(dev, I40IW_DEBUG_PBLE, "chunk_size[%d] = 0x%x vaddr=%p fpm_addr = %llx\n",
159 chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
160 return 0;
161}
162
163
164
165
166
167
168static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk)
169{
170 struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
171 int i;
172
173 if (!chunk->pg_cnt)
174 goto done;
175 for (i = 0; i < chunk->pg_cnt; i++)
176 dma_unmap_page(&pcidev->dev, chunk->dmaaddrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
177
178 done:
179 kfree(chunk->dmaaddrs);
180 chunk->dmaaddrs = NULL;
181 vfree(chunk->vaddr);
182 chunk->vaddr = NULL;
183 chunk->type = 0;
184}
185
186
187
188
189
190
191
192static enum i40iw_status_code i40iw_get_vmalloc_mem(struct i40iw_hw *hw,
193 struct i40iw_chunk *chunk,
194 int pg_cnt)
195{
196 struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
197 struct page *page;
198 u8 *addr;
199 u32 size;
200 int i;
201
202 chunk->dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
203 if (!chunk->dmaaddrs)
204 return I40IW_ERR_NO_MEMORY;
205 size = PAGE_SIZE * pg_cnt;
206 chunk->vaddr = vmalloc(size);
207 if (!chunk->vaddr) {
208 kfree(chunk->dmaaddrs);
209 chunk->dmaaddrs = NULL;
210 return I40IW_ERR_NO_MEMORY;
211 }
212 chunk->size = size;
213 addr = (u8 *)chunk->vaddr;
214 for (i = 0; i < pg_cnt; i++) {
215 page = vmalloc_to_page((void *)addr);
216 if (!page)
217 break;
218 chunk->dmaaddrs[i] = dma_map_page(&pcidev->dev, page, 0,
219 PAGE_SIZE, DMA_BIDIRECTIONAL);
220 if (dma_mapping_error(&pcidev->dev, chunk->dmaaddrs[i]))
221 break;
222 addr += PAGE_SIZE;
223 }
224
225 chunk->pg_cnt = i;
226 chunk->type = I40IW_VMALLOC;
227 if (i == pg_cnt)
228 return 0;
229
230 i40iw_free_vmalloc_mem(hw, chunk);
231 return I40IW_ERR_NO_MEMORY;
232}
233
234
235
236
237
238
239static inline u32 fpm_to_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, u64 addr)
240{
241 return (addr - (pble_rsrc->fpm_base_addr)) >> 3;
242}
243
244
245
246
247
248
249
250static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev,
251 struct i40iw_hmc_pble_rsrc *pble_rsrc,
252 struct i40iw_add_page_info *info)
253{
254 u8 *addr;
255 struct i40iw_dma_mem mem;
256 struct i40iw_hmc_pd_entry *pd_entry;
257 struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
258 struct i40iw_hmc_info *hmc_info = info->hmc_info;
259 struct i40iw_chunk *chunk = info->chunk;
260 struct i40iw_manage_vf_pble_info vf_pble_info;
261 enum i40iw_status_code status = 0;
262 u32 rel_pd_idx = info->idx.rel_pd_idx;
263 u32 pd_idx = info->idx.pd_idx;
264 u32 i;
265
266 status = i40iw_get_vmalloc_mem(dev->hw, chunk, info->pages);
267 if (status)
268 return I40IW_ERR_NO_MEMORY;
269 status = i40iw_add_sd_table_entry(dev->hw, hmc_info,
270 info->idx.sd_idx, I40IW_SD_TYPE_PAGED,
271 I40IW_HMC_DIRECT_BP_SIZE);
272 if (status)
273 goto error;
274 if (!dev->is_pf) {
275 status = i40iw_vchnl_vf_add_hmc_objs(dev, I40IW_HMC_IW_PBLE,
276 fpm_to_idx(pble_rsrc,
277 pble_rsrc->next_fpm_addr),
278 (info->pages << PBLE_512_SHIFT));
279 if (status) {
280 i40iw_pr_err("allocate PBLEs in the PF. Error %i\n", status);
281 goto error;
282 }
283 }
284 addr = chunk->vaddr;
285 for (i = 0; i < info->pages; i++) {
286 mem.pa = chunk->dmaaddrs[i];
287 mem.size = PAGE_SIZE;
288 mem.va = (void *)(addr);
289 pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
290 if (!pd_entry->valid) {
291 status = i40iw_add_pd_table_entry(dev->hw, hmc_info, pd_idx++, &mem);
292 if (status)
293 goto error;
294 addr += PAGE_SIZE;
295 } else {
296 i40iw_pr_err("pd entry is valid expecting to be invalid\n");
297 }
298 }
299 if (!dev->is_pf) {
300 vf_pble_info.first_pd_index = info->idx.rel_pd_idx;
301 vf_pble_info.inv_pd_ent = false;
302 vf_pble_info.pd_entry_cnt = PBLE_PER_PAGE;
303 vf_pble_info.pd_pl_pba = sd_entry->u.pd_table.pd_page_addr.pa;
304 vf_pble_info.sd_index = info->idx.sd_idx;
305 status = i40iw_hw_manage_vf_pble_bp(dev->back_dev,
306 &vf_pble_info, true);
307 if (status) {
308 i40iw_pr_err("CQP manage VF PBLE BP failed. %i\n", status);
309 goto error;
310 }
311 }
312 chunk->fpm_addr = pble_rsrc->next_fpm_addr;
313 return 0;
314error:
315 i40iw_free_vmalloc_mem(dev->hw, chunk);
316 return status;
317}
318
319
320
321
322
323
324static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
325 struct i40iw_hmc_pble_rsrc *pble_rsrc)
326{
327 struct i40iw_hmc_sd_entry *sd_entry;
328 struct i40iw_hmc_info *hmc_info;
329 struct i40iw_chunk *chunk;
330 struct i40iw_add_page_info info;
331 struct sd_pd_idx *idx = &info.idx;
332 enum i40iw_status_code ret_code = 0;
333 enum i40iw_sd_entry_type sd_entry_type;
334 u64 sd_reg_val = 0;
335 u32 pages;
336
337 if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
338 return I40IW_ERR_NO_MEMORY;
339 if (pble_rsrc->next_fpm_addr & 0xfff) {
340 i40iw_pr_err("next fpm_addr %llx\n", pble_rsrc->next_fpm_addr);
341 return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
342 }
343 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
344 if (!chunk)
345 return I40IW_ERR_NO_MEMORY;
346 hmc_info = dev->hmc_info;
347 chunk->fpm_addr = pble_rsrc->next_fpm_addr;
348 get_sd_pd_idx(pble_rsrc, idx);
349 sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
350 pages = (idx->rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD -
351 idx->rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD;
352 pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
353 info.chunk = chunk;
354 info.hmc_info = hmc_info;
355 info.pages = pages;
356 info.sd_entry = sd_entry;
357 if (!sd_entry->valid) {
358 sd_entry_type = (!idx->rel_pd_idx &&
359 (pages == I40IW_HMC_PD_CNT_IN_SD) &&
360 dev->is_pf) ? I40IW_SD_TYPE_DIRECT : I40IW_SD_TYPE_PAGED;
361 } else {
362 sd_entry_type = sd_entry->entry_type;
363 }
364 i40iw_debug(dev, I40IW_DEBUG_PBLE,
365 "pages = %d, unallocated_pble[%u] current_fpm_addr = %llx\n",
366 pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr);
367 i40iw_debug(dev, I40IW_DEBUG_PBLE, "sd_entry_type = %d sd_entry valid = %d\n",
368 sd_entry_type, sd_entry->valid);
369
370 if (sd_entry_type == I40IW_SD_TYPE_DIRECT)
371 ret_code = add_sd_direct(dev, pble_rsrc, &info);
372 if (ret_code)
373 sd_entry_type = I40IW_SD_TYPE_PAGED;
374 else
375 pble_rsrc->stats_direct_sds++;
376
377 if (sd_entry_type == I40IW_SD_TYPE_PAGED) {
378 ret_code = add_bp_pages(dev, pble_rsrc, &info);
379 if (ret_code)
380 goto error;
381 else
382 pble_rsrc->stats_paged_sds++;
383 }
384
385 if (gen_pool_add_virt(pble_rsrc->pinfo.pool, (unsigned long)chunk->vaddr,
386 (phys_addr_t)chunk->fpm_addr, chunk->size, -1)) {
387 i40iw_pr_err("could not allocate memory by gen_pool_addr_virt()\n");
388 ret_code = I40IW_ERR_NO_MEMORY;
389 goto error;
390 }
391 pble_rsrc->next_fpm_addr += chunk->size;
392 i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n",
393 pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
394 pble_rsrc->unallocated_pble -= (chunk->size >> 3);
395 list_add(&chunk->list, &pble_rsrc->pinfo.clist);
396 sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ?
397 sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa;
398 if (sd_entry->valid)
399 return 0;
400 if (dev->is_pf) {
401 ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id,
402 sd_reg_val, idx->sd_idx,
403 sd_entry->entry_type, true);
404 if (ret_code) {
405 i40iw_pr_err("cqp cmd failed for sd (pbles)\n");
406 goto error;
407 }
408 }
409
410 sd_entry->valid = true;
411 return 0;
412 error:
413 kfree(chunk);
414 return ret_code;
415}
416
417
418
419
420
421
422static void free_lvl2(struct i40iw_hmc_pble_rsrc *pble_rsrc,
423 struct i40iw_pble_alloc *palloc)
424{
425 u32 i;
426 struct gen_pool *pool;
427 struct i40iw_pble_level2 *lvl2 = &palloc->level2;
428 struct i40iw_pble_info *root = &lvl2->root;
429 struct i40iw_pble_info *leaf = lvl2->leaf;
430
431 pool = pble_rsrc->pinfo.pool;
432
433 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
434 if (leaf->addr)
435 gen_pool_free(pool, leaf->addr, (leaf->cnt << 3));
436 else
437 break;
438 }
439
440 if (root->addr)
441 gen_pool_free(pool, root->addr, (root->cnt << 3));
442
443 kfree(lvl2->leaf);
444 lvl2->leaf = NULL;
445}
446
447
448
449
450
451
452
453static enum i40iw_status_code get_lvl2_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
454 struct i40iw_pble_alloc *palloc,
455 struct gen_pool *pool)
456{
457 u32 lf4k, lflast, total, i;
458 u32 pblcnt = PBLE_PER_PAGE;
459 u64 *addr;
460 struct i40iw_pble_level2 *lvl2 = &palloc->level2;
461 struct i40iw_pble_info *root = &lvl2->root;
462 struct i40iw_pble_info *leaf;
463
464
465 lf4k = palloc->total_cnt >> 9;
466 lflast = palloc->total_cnt % PBLE_PER_PAGE;
467 total = (lflast == 0) ? lf4k : lf4k + 1;
468 lvl2->leaf_cnt = total;
469
470 leaf = kzalloc((sizeof(*leaf) * total), GFP_ATOMIC);
471 if (!leaf)
472 return I40IW_ERR_NO_MEMORY;
473 lvl2->leaf = leaf;
474
475 root->addr = gen_pool_alloc(pool, (total << 3));
476 if (!root->addr) {
477 kfree(lvl2->leaf);
478 lvl2->leaf = NULL;
479 return I40IW_ERR_NO_MEMORY;
480 }
481 root->idx = fpm_to_idx(pble_rsrc,
482 (u64)gen_pool_virt_to_phys(pool, root->addr));
483 root->cnt = total;
484 addr = (u64 *)root->addr;
485 for (i = 0; i < total; i++, leaf++) {
486 pblcnt = (lflast && ((i + 1) == total)) ? lflast : PBLE_PER_PAGE;
487 leaf->addr = gen_pool_alloc(pool, (pblcnt << 3));
488 if (!leaf->addr)
489 goto error;
490 leaf->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, leaf->addr));
491
492 leaf->cnt = pblcnt;
493 *addr = (u64)leaf->idx;
494 addr++;
495 }
496 palloc->level = I40IW_LEVEL_2;
497 pble_rsrc->stats_lvl2++;
498 return 0;
499 error:
500 free_lvl2(pble_rsrc, palloc);
501 return I40IW_ERR_NO_MEMORY;
502}
503
504
505
506
507
508
509
510static enum i40iw_status_code get_lvl1_pble(struct i40iw_sc_dev *dev,
511 struct i40iw_hmc_pble_rsrc *pble_rsrc,
512 struct i40iw_pble_alloc *palloc)
513{
514 u64 *addr;
515 struct gen_pool *pool;
516 struct i40iw_pble_info *lvl1 = &palloc->level1;
517
518 pool = pble_rsrc->pinfo.pool;
519 addr = (u64 *)gen_pool_alloc(pool, (palloc->total_cnt << 3));
520
521 if (!addr)
522 return I40IW_ERR_NO_MEMORY;
523
524 palloc->level = I40IW_LEVEL_1;
525 lvl1->addr = (unsigned long)addr;
526 lvl1->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool,
527 (unsigned long)addr));
528 lvl1->cnt = palloc->total_cnt;
529 pble_rsrc->stats_lvl1++;
530 return 0;
531}
532
533
534
535
536
537
538
539
540static inline enum i40iw_status_code get_lvl1_lvl2_pble(struct i40iw_sc_dev *dev,
541 struct i40iw_hmc_pble_rsrc *pble_rsrc,
542 struct i40iw_pble_alloc *palloc,
543 struct gen_pool *pool)
544{
545 enum i40iw_status_code status = 0;
546
547 status = get_lvl1_pble(dev, pble_rsrc, palloc);
548 if (status && (palloc->total_cnt > PBLE_PER_PAGE))
549 status = get_lvl2_pble(pble_rsrc, palloc, pool);
550 return status;
551}
552
553
554
555
556
557
558
559
560enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev,
561 struct i40iw_hmc_pble_rsrc *pble_rsrc,
562 struct i40iw_pble_alloc *palloc,
563 u32 pble_cnt)
564{
565 struct gen_pool *pool;
566 enum i40iw_status_code status = 0;
567 u32 max_sds = 0;
568 int i;
569
570 pool = pble_rsrc->pinfo.pool;
571 palloc->total_cnt = pble_cnt;
572 palloc->level = I40IW_LEVEL_0;
573
574 status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
575 if (!status)
576 goto exit;
577 max_sds = (palloc->total_cnt >> 18) + 1;
578 for (i = 0; i < max_sds; i++) {
579 status = add_pble_pool(dev, pble_rsrc);
580 if (status)
581 break;
582 status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
583 if (!status)
584 break;
585 }
586exit:
587 if (!status)
588 pble_rsrc->stats_alloc_ok++;
589 else
590 pble_rsrc->stats_alloc_fail++;
591
592 return status;
593}
594
595
596
597
598
599
600void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
601 struct i40iw_pble_alloc *palloc)
602{
603 struct gen_pool *pool;
604
605 pool = pble_rsrc->pinfo.pool;
606 if (palloc->level == I40IW_LEVEL_2)
607 free_lvl2(pble_rsrc, palloc);
608 else
609 gen_pool_free(pool, palloc->level1.addr,
610 (palloc->level1.cnt << 3));
611 pble_rsrc->stats_alloc_freed++;
612}
613