1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#define NVKM_VMM_LEVELS_MAX 5
23#include "vmm.h"
24
25#include <subdev/fb.h>
26
27static void
28nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
29{
30 struct nvkm_vmm_pt *pgt = *ppgt;
31 if (pgt) {
32 kvfree(pgt->pde);
33 kfree(pgt);
34 *ppgt = NULL;
35 }
36}
37
38
39static struct nvkm_vmm_pt *
40nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
41 const struct nvkm_vmm_page *page)
42{
43 const u32 pten = 1 << desc->bits;
44 struct nvkm_vmm_pt *pgt;
45 u32 lpte = 0;
46
47 if (desc->type > PGT) {
48 if (desc->type == SPT) {
49 const struct nvkm_vmm_desc *pair = page[-1].desc;
50 lpte = pten >> (desc->bits - pair->bits);
51 } else {
52 lpte = pten;
53 }
54 }
55
56 if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
57 return NULL;
58 pgt->page = page ? page->shift : 0;
59 pgt->sparse = sparse;
60
61 if (desc->type == PGD) {
62 pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
63 if (!pgt->pde) {
64 kfree(pgt);
65 return NULL;
66 }
67 }
68
69 return pgt;
70}
71
72struct nvkm_vmm_iter {
73 const struct nvkm_vmm_page *page;
74 const struct nvkm_vmm_desc *desc;
75 struct nvkm_vmm *vmm;
76 u64 cnt;
77 u16 max, lvl;
78 u32 pte[NVKM_VMM_LEVELS_MAX];
79 struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
80 int flush;
81};
82
83#ifdef CONFIG_NOUVEAU_DEBUG_MMU
84static const char *
85nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
86{
87 switch (desc->type) {
88 case PGD: return "PGD";
89 case PGT: return "PGT";
90 case SPT: return "SPT";
91 case LPT: return "LPT";
92 default:
93 return "UNKNOWN";
94 }
95}
96
97static void
98nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
99{
100 int lvl;
101 for (lvl = it->max; lvl >= 0; lvl--) {
102 if (lvl >= it->lvl)
103 buf += sprintf(buf, "%05x:", it->pte[lvl]);
104 else
105 buf += sprintf(buf, "xxxxx:");
106 }
107}
108
109#define TRA(i,f,a...) do { \
110 char _buf[NVKM_VMM_LEVELS_MAX * 7]; \
111 struct nvkm_vmm_iter *_it = (i); \
112 nvkm_vmm_trace(_it, _buf); \
113 VMM_TRACE(_it->vmm, "%s "f, _buf, ##a); \
114} while(0)
115#else
116#define TRA(i,f,a...)
117#endif
118
119static inline void
120nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
121{
122 it->flush = min(it->flush, it->max - it->lvl);
123}
124
125static inline void
126nvkm_vmm_flush(struct nvkm_vmm_iter *it)
127{
128 if (it->flush != NVKM_VMM_LEVELS_MAX) {
129 if (it->vmm->func->flush) {
130 TRA(it, "flush: %d", it->flush);
131 it->vmm->func->flush(it->vmm, it->flush);
132 }
133 it->flush = NVKM_VMM_LEVELS_MAX;
134 }
135}
136
137static void
138nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
139{
140 const struct nvkm_vmm_desc *desc = it->desc;
141 const int type = desc[it->lvl].type == SPT;
142 struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
143 struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
144 struct nvkm_mmu_pt *pt = pgt->pt[type];
145 struct nvkm_vmm *vmm = it->vmm;
146 u32 pdei = it->pte[it->lvl + 1];
147
148
149 it->lvl++;
150 if (--pgd->refs[0]) {
151 const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
152
153 TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
154 pgt->pt[type] = NULL;
155 if (!pgt->refs[!type]) {
156
157 if (pgd->pt[0]) {
158 if (pgt->sparse) {
159 func->sparse(vmm, pgd->pt[0], pdei, 1);
160 pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
161 } else {
162 func->unmap(vmm, pgd->pt[0], pdei, 1);
163 pgd->pde[pdei] = NULL;
164 }
165 } else {
166
167
168
169
170 func->pde(vmm, pgd, pdei);
171 pgd->pde[pdei] = NULL;
172 }
173 } else {
174
175
176
177 func->pde(vmm, pgd, pdei);
178 }
179
180
181 nvkm_vmm_flush_mark(it);
182 nvkm_vmm_flush(it);
183 } else {
184
185 nvkm_vmm_unref_pdes(it);
186 }
187
188
189 TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
190 nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
191 if (!pgt->refs[!type])
192 nvkm_vmm_pt_del(&pgt);
193 it->lvl--;
194}
195
196static void
197nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
198 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
199{
200 const struct nvkm_vmm_desc *pair = it->page[-1].desc;
201 const u32 sptb = desc->bits - pair->bits;
202 const u32 sptn = 1 << sptb;
203 struct nvkm_vmm *vmm = it->vmm;
204 u32 spti = ptei & (sptn - 1), lpti, pteb;
205
206
207
208
209 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
210 const u32 pten = min(sptn - spti, ptes);
211 pgt->pte[lpti] -= pten;
212 ptes -= pten;
213 }
214
215
216 if (!pgt->refs[0])
217 return;
218
219 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
220
221 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
222 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
223 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
224 break;
225 }
226 continue;
227 }
228
229
230
231
232
233
234
235 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
236 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
237 if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
238 break;
239 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
240 }
241
242 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
243 TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
244 pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
245 } else
246 if (pair->func->invalid) {
247
248
249
250
251 TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
252 pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
253 }
254 }
255}
256
257static bool
258nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
259{
260 const struct nvkm_vmm_desc *desc = it->desc;
261 const int type = desc->type == SPT;
262 struct nvkm_vmm_pt *pgt = it->pt[0];
263 bool dma;
264
265 if (pfn) {
266
267 dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
268 if (dma) {
269
270 nvkm_vmm_flush_mark(it);
271 nvkm_vmm_flush(it);
272 desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
273 }
274 }
275
276
277 pgt->refs[type] -= ptes;
278
279
280 if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
281 nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
282
283
284 if (!pgt->refs[type]) {
285 it->lvl++;
286 TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
287 it->lvl--;
288 nvkm_vmm_unref_pdes(it);
289 return false;
290 }
291
292 return true;
293}
294
295static void
296nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
297 const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
298{
299 const struct nvkm_vmm_desc *pair = it->page[-1].desc;
300 const u32 sptb = desc->bits - pair->bits;
301 const u32 sptn = 1 << sptb;
302 struct nvkm_vmm *vmm = it->vmm;
303 u32 spti = ptei & (sptn - 1), lpti, pteb;
304
305
306
307
308 for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
309 const u32 pten = min(sptn - spti, ptes);
310 pgt->pte[lpti] += pten;
311 ptes -= pten;
312 }
313
314
315 if (!pgt->refs[0])
316 return;
317
318 for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
319
320 if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
321 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
322 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
323 break;
324 }
325 continue;
326 }
327
328
329
330
331
332
333
334 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
335 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
336 if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
337 break;
338 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
339 }
340
341 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
342 const u32 spti = pteb * sptn;
343 const u32 sptc = ptes * sptn;
344
345
346
347 TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
348 desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
349
350 TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
351 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
352 } else
353 if (pair->func->invalid) {
354
355
356
357 TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
358 pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
359 }
360 }
361}
362
363static bool
364nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
365{
366 const struct nvkm_vmm_desc *desc = it->desc;
367 const int type = desc->type == SPT;
368 struct nvkm_vmm_pt *pgt = it->pt[0];
369
370
371 pgt->refs[type] += ptes;
372
373
374 if (desc->type == SPT)
375 nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
376
377 return true;
378}
379
380static void
381nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
382 struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
383{
384 if (desc->type == PGD) {
385 while (ptes--)
386 pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
387 } else
388 if (desc->type == LPT) {
389 memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
390 }
391}
392
393static bool
394nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
395{
396 struct nvkm_vmm_pt *pt = it->pt[0];
397 if (it->desc->type == PGD)
398 memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
399 else
400 if (it->desc->type == LPT)
401 memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
402 return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
403}
404
405static bool
406nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
407{
408 nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
409 return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
410}
411
412static bool
413nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
414{
415 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
416 const int type = desc->type == SPT;
417 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
418 const bool zero = !pgt->sparse && !desc->func->invalid;
419 struct nvkm_vmm *vmm = it->vmm;
420 struct nvkm_mmu *mmu = vmm->mmu;
421 struct nvkm_mmu_pt *pt;
422 u32 pten = 1 << desc->bits;
423 u32 pteb, ptei, ptes;
424 u32 size = desc->size * pten;
425
426 pgd->refs[0]++;
427
428 pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
429 if (!pgt->pt[type]) {
430 it->lvl--;
431 nvkm_vmm_unref_pdes(it);
432 return false;
433 }
434
435 if (zero)
436 goto done;
437
438 pt = pgt->pt[type];
439
440 if (desc->type == LPT && pgt->refs[1]) {
441
442
443
444
445
446
447 for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
448 bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
449 for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
450 bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
451 if (spte != next)
452 break;
453 }
454
455 if (!spte) {
456 if (pgt->sparse)
457 desc->func->sparse(vmm, pt, pteb, ptes);
458 else
459 desc->func->invalid(vmm, pt, pteb, ptes);
460 memset(&pgt->pte[pteb], 0x00, ptes);
461 } else {
462 desc->func->unmap(vmm, pt, pteb, ptes);
463 while (ptes--)
464 pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
465 }
466 }
467 } else {
468 if (pgt->sparse) {
469 nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
470 desc->func->sparse(vmm, pt, 0, pten);
471 } else {
472 desc->func->invalid(vmm, pt, 0, pten);
473 }
474 }
475
476done:
477 TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
478 it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
479 nvkm_vmm_flush_mark(it);
480 return true;
481}
482
483static bool
484nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
485{
486 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
487 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
488
489 pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
490 if (!pgt) {
491 if (!pgd->refs[0])
492 nvkm_vmm_unref_pdes(it);
493 return false;
494 }
495
496 pgd->pde[pdei] = pgt;
497 return true;
498}
499
500static inline u64
501nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
502 u64 addr, u64 size, const char *name, bool ref, bool pfn,
503 bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
504 nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
505 nvkm_vmm_pxe_func CLR_PTES)
506{
507 const struct nvkm_vmm_desc *desc = page->desc;
508 struct nvkm_vmm_iter it;
509 u64 bits = addr >> page->shift;
510
511 it.page = page;
512 it.desc = desc;
513 it.vmm = vmm;
514 it.cnt = size >> page->shift;
515 it.flush = NVKM_VMM_LEVELS_MAX;
516
517
518 for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
519 it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
520 bits >>= desc[it.lvl].bits;
521 }
522 it.max = --it.lvl;
523 it.pt[it.max] = vmm->pd;
524
525 it.lvl = 0;
526 TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
527 addr, size, page->shift, it.cnt);
528 it.lvl = it.max;
529
530
531 while (it.cnt) {
532 struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
533 const int type = desc->type == SPT;
534 const u32 pten = 1 << desc->bits;
535 const u32 ptei = it.pte[0];
536 const u32 ptes = min_t(u64, it.cnt, pten - ptei);
537
538
539 for (; it.lvl; it.lvl--) {
540 const u32 pdei = it.pte[it.lvl];
541 struct nvkm_vmm_pt *pgd = pgt;
542
543
544 if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
545 if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
546 goto fail;
547 }
548 it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
549
550
551
552
553
554
555
556 if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
557 if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
558 goto fail;
559 }
560 }
561
562
563 if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
564 struct nvkm_mmu_pt *pt = pgt->pt[type];
565 if (MAP_PTES || CLR_PTES) {
566 if (MAP_PTES)
567 MAP_PTES(vmm, pt, ptei, ptes, map);
568 else
569 CLR_PTES(vmm, pt, ptei, ptes);
570 nvkm_vmm_flush_mark(&it);
571 }
572 }
573
574
575 it.pte[it.lvl] += ptes;
576 it.cnt -= ptes;
577 if (it.cnt) {
578 while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
579 it.pte[it.lvl++] = 0;
580 it.pte[it.lvl]++;
581 }
582 }
583 }
584
585 nvkm_vmm_flush(&it);
586 return ~0ULL;
587
588fail:
589
590
591
592 addr = it.pte[it.max--];
593 do {
594 addr = addr << desc[it.max].bits;
595 addr |= it.pte[it.max];
596 } while (it.max--);
597
598 return addr << page->shift;
599}
600
601static void
602nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
603 u64 addr, u64 size)
604{
605 nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
606 nvkm_vmm_sparse_unref_ptes, NULL, NULL,
607 page->desc->func->invalid ?
608 page->desc->func->invalid : page->desc->func->unmap);
609}
610
611static int
612nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
613 u64 addr, u64 size)
614{
615 if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
616 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
617 true, false, nvkm_vmm_sparse_ref_ptes,
618 NULL, NULL, page->desc->func->sparse);
619 if (fail != ~0ULL) {
620 if ((size = fail - addr))
621 nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
622 return -ENOMEM;
623 }
624 return 0;
625 }
626 return -EINVAL;
627}
628
629static int
630nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
631{
632 const struct nvkm_vmm_page *page = vmm->func->page;
633 int m = 0, i;
634 u64 start = addr;
635 u64 block;
636
637 while (size) {
638
639 while (size < (1ULL << page[m].shift))
640 m++;
641 i = m;
642
643
644 while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
645 i++;
646
647
648 if (i != m) {
649
650 u64 next = 1ULL << page[i - 1].shift;
651 u64 part = ALIGN(addr, next) - addr;
652 if (size - part >= next)
653 block = (part >> page[i].shift) << page[i].shift;
654 else
655 block = (size >> page[i].shift) << page[i].shift;
656 } else {
657 block = (size >> page[i].shift) << page[i].shift;
658 }
659
660
661 if (ref) {
662 int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
663 if (ret) {
664 if ((size = addr - start))
665 nvkm_vmm_ptes_sparse(vmm, start, size, false);
666 return ret;
667 }
668 } else {
669 nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
670 }
671
672 size -= block;
673 addr += block;
674 }
675
676 return 0;
677}
678
679static void
680nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
681 u64 addr, u64 size, bool sparse, bool pfn)
682{
683 const struct nvkm_vmm_desc_func *func = page->desc->func;
684 nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
685 false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
686 sparse ? func->sparse : func->invalid ? func->invalid :
687 func->unmap);
688}
689
690static int
691nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
692 u64 addr, u64 size, struct nvkm_vmm_map *map,
693 nvkm_vmm_pte_func func)
694{
695 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
696 false, nvkm_vmm_ref_ptes, func, map, NULL);
697 if (fail != ~0ULL) {
698 if ((size = fail - addr))
699 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
700 return -ENOMEM;
701 }
702 return 0;
703}
704
705static void
706nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
707 u64 addr, u64 size, bool sparse, bool pfn)
708{
709 const struct nvkm_vmm_desc_func *func = page->desc->func;
710 nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
711 NULL, NULL, NULL,
712 sparse ? func->sparse : func->invalid ? func->invalid :
713 func->unmap);
714}
715
716static void
717nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
718 u64 addr, u64 size, struct nvkm_vmm_map *map,
719 nvkm_vmm_pte_func func)
720{
721 nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
722 NULL, func, map, NULL);
723}
724
725static void
726nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
727 u64 addr, u64 size)
728{
729 nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
730 nvkm_vmm_unref_ptes, NULL, NULL, NULL);
731}
732
733static int
734nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
735 u64 addr, u64 size)
736{
737 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
738 nvkm_vmm_ref_ptes, NULL, NULL, NULL);
739 if (fail != ~0ULL) {
740 if (fail != addr)
741 nvkm_vmm_ptes_put(vmm, page, addr, fail - addr);
742 return -ENOMEM;
743 }
744 return 0;
745}
746
747static inline struct nvkm_vma *
748nvkm_vma_new(u64 addr, u64 size)
749{
750 struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
751 if (vma) {
752 vma->addr = addr;
753 vma->size = size;
754 vma->page = NVKM_VMA_PAGE_NONE;
755 vma->refd = NVKM_VMA_PAGE_NONE;
756 }
757 return vma;
758}
759
760struct nvkm_vma *
761nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
762{
763 struct nvkm_vma *new;
764
765 BUG_ON(vma->size == tail);
766
767 if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
768 return NULL;
769 vma->size -= tail;
770
771 new->mapref = vma->mapref;
772 new->sparse = vma->sparse;
773 new->page = vma->page;
774 new->refd = vma->refd;
775 new->used = vma->used;
776 new->part = vma->part;
777 new->busy = vma->busy;
778 new->mapped = vma->mapped;
779 list_add(&new->head, &vma->head);
780 return new;
781}
782
783static inline void
784nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
785{
786 rb_erase(&vma->tree, &vmm->free);
787}
788
789static inline void
790nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
791{
792 nvkm_vmm_free_remove(vmm, vma);
793 list_del(&vma->head);
794 kfree(vma);
795}
796
797static void
798nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
799{
800 struct rb_node **ptr = &vmm->free.rb_node;
801 struct rb_node *parent = NULL;
802
803 while (*ptr) {
804 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
805 parent = *ptr;
806 if (vma->size < this->size)
807 ptr = &parent->rb_left;
808 else
809 if (vma->size > this->size)
810 ptr = &parent->rb_right;
811 else
812 if (vma->addr < this->addr)
813 ptr = &parent->rb_left;
814 else
815 if (vma->addr > this->addr)
816 ptr = &parent->rb_right;
817 else
818 BUG();
819 }
820
821 rb_link_node(&vma->tree, parent, ptr);
822 rb_insert_color(&vma->tree, &vmm->free);
823}
824
825static inline void
826nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
827{
828 rb_erase(&vma->tree, &vmm->root);
829}
830
831static inline void
832nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
833{
834 nvkm_vmm_node_remove(vmm, vma);
835 list_del(&vma->head);
836 kfree(vma);
837}
838
839static void
840nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
841{
842 struct rb_node **ptr = &vmm->root.rb_node;
843 struct rb_node *parent = NULL;
844
845 while (*ptr) {
846 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
847 parent = *ptr;
848 if (vma->addr < this->addr)
849 ptr = &parent->rb_left;
850 else
851 if (vma->addr > this->addr)
852 ptr = &parent->rb_right;
853 else
854 BUG();
855 }
856
857 rb_link_node(&vma->tree, parent, ptr);
858 rb_insert_color(&vma->tree, &vmm->root);
859}
860
861struct nvkm_vma *
862nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
863{
864 struct rb_node *node = vmm->root.rb_node;
865 while (node) {
866 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
867 if (addr < vma->addr)
868 node = node->rb_left;
869 else
870 if (addr >= vma->addr + vma->size)
871 node = node->rb_right;
872 else
873 return vma;
874 }
875 return NULL;
876}
877
878#define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL : \
879 list_entry((root)->head.dir, struct nvkm_vma, head))
880
881static struct nvkm_vma *
882nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev,
883 struct nvkm_vma *vma, struct nvkm_vma *next, u64 size)
884{
885 if (next) {
886 if (vma->size == size) {
887 vma->size += next->size;
888 nvkm_vmm_node_delete(vmm, next);
889 if (prev) {
890 prev->size += vma->size;
891 nvkm_vmm_node_delete(vmm, vma);
892 return prev;
893 }
894 return vma;
895 }
896 BUG_ON(prev);
897
898 nvkm_vmm_node_remove(vmm, next);
899 vma->size -= size;
900 next->addr -= size;
901 next->size += size;
902 nvkm_vmm_node_insert(vmm, next);
903 return next;
904 }
905
906 if (prev) {
907 if (vma->size != size) {
908 nvkm_vmm_node_remove(vmm, vma);
909 prev->size += size;
910 vma->addr += size;
911 vma->size -= size;
912 nvkm_vmm_node_insert(vmm, vma);
913 } else {
914 prev->size += vma->size;
915 nvkm_vmm_node_delete(vmm, vma);
916 }
917 return prev;
918 }
919
920 return vma;
921}
922
923struct nvkm_vma *
924nvkm_vmm_node_split(struct nvkm_vmm *vmm,
925 struct nvkm_vma *vma, u64 addr, u64 size)
926{
927 struct nvkm_vma *prev = NULL;
928
929 if (vma->addr != addr) {
930 prev = vma;
931 if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr)))
932 return NULL;
933 vma->part = true;
934 nvkm_vmm_node_insert(vmm, vma);
935 }
936
937 if (vma->size != size) {
938 struct nvkm_vma *tmp;
939 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
940 nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size);
941 return NULL;
942 }
943 tmp->part = true;
944 nvkm_vmm_node_insert(vmm, tmp);
945 }
946
947 return vma;
948}
949
950static void
951nvkm_vma_dump(struct nvkm_vma *vma)
952{
953 printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c %p\n",
954 vma->addr, (u64)vma->size,
955 vma->used ? '-' : 'F',
956 vma->mapref ? 'R' : '-',
957 vma->sparse ? 'S' : '-',
958 vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
959 vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
960 vma->part ? 'P' : '-',
961 vma->busy ? 'B' : '-',
962 vma->mapped ? 'M' : '-',
963 vma->memory);
964}
965
966static void
967nvkm_vmm_dump(struct nvkm_vmm *vmm)
968{
969 struct nvkm_vma *vma;
970 list_for_each_entry(vma, &vmm->list, head) {
971 nvkm_vma_dump(vma);
972 }
973}
974
975static void
976nvkm_vmm_dtor(struct nvkm_vmm *vmm)
977{
978 struct nvkm_vma *vma;
979 struct rb_node *node;
980
981 if (0)
982 nvkm_vmm_dump(vmm);
983
984 while ((node = rb_first(&vmm->root))) {
985 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
986 nvkm_vmm_put(vmm, &vma);
987 }
988
989 if (vmm->bootstrapped) {
990 const struct nvkm_vmm_page *page = vmm->func->page;
991 const u64 limit = vmm->limit - vmm->start;
992
993 while (page[1].shift)
994 page++;
995
996 nvkm_mmu_ptc_dump(vmm->mmu);
997 nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
998 }
999
1000 vma = list_first_entry(&vmm->list, typeof(*vma), head);
1001 list_del(&vma->head);
1002 kfree(vma);
1003 WARN_ON(!list_empty(&vmm->list));
1004
1005 if (vmm->nullp) {
1006 dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
1007 vmm->nullp, vmm->null);
1008 }
1009
1010 if (vmm->pd) {
1011 nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
1012 nvkm_vmm_pt_del(&vmm->pd);
1013 }
1014}
1015
1016static int
1017nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size)
1018{
1019 struct nvkm_vma *vma;
1020 if (!(vma = nvkm_vma_new(addr, size)))
1021 return -ENOMEM;
1022 vma->mapref = true;
1023 vma->sparse = false;
1024 vma->used = true;
1025 nvkm_vmm_node_insert(vmm, vma);
1026 list_add_tail(&vma->head, &vmm->list);
1027 return 0;
1028}
1029
1030static int
1031nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1032 u32 pd_header, bool managed, u64 addr, u64 size,
1033 struct lock_class_key *key, const char *name,
1034 struct nvkm_vmm *vmm)
1035{
1036 static struct lock_class_key _key;
1037 const struct nvkm_vmm_page *page = func->page;
1038 const struct nvkm_vmm_desc *desc;
1039 struct nvkm_vma *vma;
1040 int levels, bits = 0, ret;
1041
1042 vmm->func = func;
1043 vmm->mmu = mmu;
1044 vmm->name = name;
1045 vmm->debug = mmu->subdev.debug;
1046 kref_init(&vmm->kref);
1047
1048 __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key);
1049
1050
1051
1052
1053 while (page[1].shift)
1054 page++;
1055
1056
1057
1058
1059
1060 for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
1061 bits += desc->bits;
1062 bits += page->shift;
1063 desc--;
1064
1065 if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
1066 return -EINVAL;
1067
1068
1069 vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
1070 if (!vmm->pd)
1071 return -ENOMEM;
1072 vmm->pd->refs[0] = 1;
1073 INIT_LIST_HEAD(&vmm->join);
1074
1075
1076
1077
1078 if (desc->size) {
1079 const u32 size = pd_header + desc->size * (1 << desc->bits);
1080 vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
1081 if (!vmm->pd->pt[0])
1082 return -ENOMEM;
1083 }
1084
1085
1086 INIT_LIST_HEAD(&vmm->list);
1087 vmm->free = RB_ROOT;
1088 vmm->root = RB_ROOT;
1089
1090 if (managed) {
1091
1092
1093
1094
1095 vmm->start = 0;
1096 vmm->limit = 1ULL << bits;
1097 if (addr + size < addr || addr + size > vmm->limit)
1098 return -EINVAL;
1099
1100
1101 if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr)))
1102 return ret;
1103
1104
1105 if (size) {
1106 if (!(vma = nvkm_vma_new(addr, size)))
1107 return -ENOMEM;
1108 nvkm_vmm_free_insert(vmm, vma);
1109 list_add_tail(&vma->head, &vmm->list);
1110 }
1111
1112
1113 addr = addr + size;
1114 size = vmm->limit - addr;
1115 if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size)))
1116 return ret;
1117 } else {
1118
1119
1120
1121 vmm->start = addr;
1122 vmm->limit = size ? (addr + size) : (1ULL << bits);
1123 if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
1124 return -EINVAL;
1125
1126 if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
1127 return -ENOMEM;
1128
1129 nvkm_vmm_free_insert(vmm, vma);
1130 list_add(&vma->head, &vmm->list);
1131 }
1132
1133 return 0;
1134}
1135
1136int
1137nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1138 u32 hdr, bool managed, u64 addr, u64 size,
1139 struct lock_class_key *key, const char *name,
1140 struct nvkm_vmm **pvmm)
1141{
1142 if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
1143 return -ENOMEM;
1144 return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
1145}
1146
1147static struct nvkm_vma *
1148nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1149 u64 addr, u64 size, u8 page, bool map)
1150{
1151 struct nvkm_vma *prev = NULL;
1152 struct nvkm_vma *next = NULL;
1153
1154 if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
1155 if (prev->memory || prev->mapped != map)
1156 prev = NULL;
1157 }
1158
1159 if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
1160 if (!next->part ||
1161 next->memory || next->mapped != map)
1162 next = NULL;
1163 }
1164
1165 if (prev || next)
1166 return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
1167 return nvkm_vmm_node_split(vmm, vma, addr, size);
1168}
1169
1170int
1171nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
1172{
1173 struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
1174 struct nvkm_vma *next;
1175 u64 limit = addr + size;
1176 u64 start = addr;
1177
1178 if (!vma)
1179 return -EINVAL;
1180
1181 do {
1182 if (!vma->mapped || vma->memory)
1183 continue;
1184
1185 size = min(limit - start, vma->size - (start - vma->addr));
1186
1187 nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
1188 start, size, false, true);
1189
1190 next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
1191 if (!WARN_ON(!next)) {
1192 vma = next;
1193 vma->refd = NVKM_VMA_PAGE_NONE;
1194 vma->mapped = false;
1195 }
1196 } while ((vma = node(vma, next)) && (start = vma->addr) < limit);
1197
1198 return 0;
1199}
1200
1201
1202
1203
1204
1205
1206int
1207nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
1208{
1209 const struct nvkm_vmm_page *page = vmm->func->page;
1210 struct nvkm_vma *vma, *tmp;
1211 u64 limit = addr + size;
1212 u64 start = addr;
1213 int pm = size >> shift;
1214 int pi = 0;
1215
1216
1217
1218
1219 while (page->shift && (page->shift != shift ||
1220 page->desc->func->pfn == NULL))
1221 page++;
1222
1223 if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
1224 !IS_ALIGNED(size, 1ULL << shift) ||
1225 addr + size < addr || addr + size > vmm->limit) {
1226 VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
1227 shift, page->shift, addr, size);
1228 return -EINVAL;
1229 }
1230
1231 if (!(vma = nvkm_vmm_node_search(vmm, addr)))
1232 return -ENOENT;
1233
1234 do {
1235 bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
1236 bool mapped = vma->mapped;
1237 u64 size = limit - start;
1238 u64 addr = start;
1239 int pn, ret = 0;
1240
1241
1242
1243
1244 for (pn = 0; pi + pn < pm; pn++) {
1245 if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
1246 break;
1247 }
1248 size = min_t(u64, size, pn << page->shift);
1249 size = min_t(u64, size, vma->size + vma->addr - addr);
1250
1251
1252
1253
1254 if (!vma->mapref || vma->memory) {
1255 ret = -EINVAL;
1256 goto next;
1257 }
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270 if (map != mapped) {
1271 tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
1272 page -
1273 vmm->func->page, map);
1274 if (WARN_ON(!tmp)) {
1275 ret = -ENOMEM;
1276 goto next;
1277 }
1278
1279 if ((tmp->mapped = map))
1280 tmp->refd = page - vmm->func->page;
1281 else
1282 tmp->refd = NVKM_VMA_PAGE_NONE;
1283 vma = tmp;
1284 }
1285
1286
1287 if (map) {
1288 struct nvkm_vmm_map args;
1289 args.page = page;
1290 args.pfn = &pfn[pi];
1291
1292 if (!mapped) {
1293 ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
1294 size, &args, page->
1295 desc->func->pfn);
1296 } else {
1297 nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
1298 page->desc->func->pfn);
1299 }
1300 } else {
1301 if (mapped) {
1302 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
1303 false, true);
1304 }
1305 }
1306
1307next:
1308
1309 if (vma->addr + vma->size == addr + size)
1310 vma = node(vma, next);
1311 start += size;
1312
1313 if (ret) {
1314
1315
1316
1317 while (size) {
1318 pfn[pi++] = NVKM_VMM_PFN_NONE;
1319 size -= 1 << page->shift;
1320 }
1321 } else {
1322 pi += size >> page->shift;
1323 }
1324 } while (vma && start < limit);
1325
1326 return 0;
1327}
1328
1329void
1330nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1331{
1332 struct nvkm_vma *prev = NULL;
1333 struct nvkm_vma *next;
1334
1335 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1336 nvkm_memory_unref(&vma->memory);
1337 vma->mapped = false;
1338
1339 if (vma->part && (prev = node(vma, prev)) && prev->mapped)
1340 prev = NULL;
1341 if ((next = node(vma, next)) && (!next->part || next->mapped))
1342 next = NULL;
1343 nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
1344}
1345
1346void
1347nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
1348{
1349 const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
1350
1351 if (vma->mapref) {
1352 nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1353 vma->refd = NVKM_VMA_PAGE_NONE;
1354 } else {
1355 nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1356 }
1357
1358 nvkm_vmm_unmap_region(vmm, vma);
1359}
1360
1361void
1362nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1363{
1364 if (vma->memory) {
1365 mutex_lock(&vmm->mutex);
1366 nvkm_vmm_unmap_locked(vmm, vma, false);
1367 mutex_unlock(&vmm->mutex);
1368 }
1369}
1370
1371static int
1372nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1373 void *argv, u32 argc, struct nvkm_vmm_map *map)
1374{
1375 switch (nvkm_memory_target(map->memory)) {
1376 case NVKM_MEM_TARGET_VRAM:
1377 if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1378 VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1379 return -EINVAL;
1380 }
1381 break;
1382 case NVKM_MEM_TARGET_HOST:
1383 case NVKM_MEM_TARGET_NCOH:
1384 if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1385 VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1386 return -EINVAL;
1387 }
1388 break;
1389 default:
1390 WARN_ON(1);
1391 return -ENOSYS;
1392 }
1393
1394 if (!IS_ALIGNED( vma->addr, 1ULL << map->page->shift) ||
1395 !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1396 !IS_ALIGNED( map->offset, 1ULL << map->page->shift) ||
1397 nvkm_memory_page(map->memory) < map->page->shift) {
1398 VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1399 vma->addr, (u64)vma->size, map->offset, map->page->shift,
1400 nvkm_memory_page(map->memory));
1401 return -EINVAL;
1402 }
1403
1404 return vmm->func->valid(vmm, argv, argc, map);
1405}
1406
1407static int
1408nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1409 void *argv, u32 argc, struct nvkm_vmm_map *map)
1410{
1411 for (map->page = vmm->func->page; map->page->shift; map->page++) {
1412 VMM_DEBUG(vmm, "trying %d", map->page->shift);
1413 if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1414 return 0;
1415 }
1416 return -EINVAL;
1417}
1418
1419static int
1420nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1421 void *argv, u32 argc, struct nvkm_vmm_map *map)
1422{
1423 nvkm_vmm_pte_func func;
1424 int ret;
1425
1426
1427 if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1428 VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1429 nvkm_memory_size(map->memory),
1430 map->offset, (u64)vma->size);
1431 return -EINVAL;
1432 }
1433
1434
1435 if (vma->page == NVKM_VMA_PAGE_NONE &&
1436 vma->refd == NVKM_VMA_PAGE_NONE) {
1437
1438 const u32 debug = vmm->debug;
1439 vmm->debug = 0;
1440 ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1441 vmm->debug = debug;
1442 if (ret) {
1443 VMM_DEBUG(vmm, "invalid at any page size");
1444 nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1445 return -EINVAL;
1446 }
1447 } else {
1448
1449 if (vma->refd != NVKM_VMA_PAGE_NONE)
1450 map->page = &vmm->func->page[vma->refd];
1451 else
1452 map->page = &vmm->func->page[vma->page];
1453
1454 ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1455 if (ret) {
1456 VMM_DEBUG(vmm, "invalid %d\n", ret);
1457 return ret;
1458 }
1459 }
1460
1461
1462 map->off = map->offset;
1463 if (map->mem) {
1464 for (; map->off; map->mem = map->mem->next) {
1465 u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1466 if (size > map->off)
1467 break;
1468 map->off -= size;
1469 }
1470 func = map->page->desc->func->mem;
1471 } else
1472 if (map->sgl) {
1473 for (; map->off; map->sgl = sg_next(map->sgl)) {
1474 u64 size = sg_dma_len(map->sgl);
1475 if (size > map->off)
1476 break;
1477 map->off -= size;
1478 }
1479 func = map->page->desc->func->sgl;
1480 } else {
1481 map->dma += map->offset >> PAGE_SHIFT;
1482 map->off = map->offset & PAGE_MASK;
1483 func = map->page->desc->func->dma;
1484 }
1485
1486
1487 if (vma->refd == NVKM_VMA_PAGE_NONE) {
1488 ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1489 if (ret)
1490 return ret;
1491
1492 vma->refd = map->page - vmm->func->page;
1493 } else {
1494 nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1495 }
1496
1497 nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1498 nvkm_memory_unref(&vma->memory);
1499 vma->memory = nvkm_memory_ref(map->memory);
1500 vma->mapped = true;
1501 vma->tags = map->tags;
1502 return 0;
1503}
1504
1505int
1506nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1507 struct nvkm_vmm_map *map)
1508{
1509 int ret;
1510 mutex_lock(&vmm->mutex);
1511 ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1512 vma->busy = false;
1513 mutex_unlock(&vmm->mutex);
1514 return ret;
1515}
1516
1517static void
1518nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1519{
1520 struct nvkm_vma *prev, *next;
1521
1522 if ((prev = node(vma, prev)) && !prev->used) {
1523 vma->addr = prev->addr;
1524 vma->size += prev->size;
1525 nvkm_vmm_free_delete(vmm, prev);
1526 }
1527
1528 if ((next = node(vma, next)) && !next->used) {
1529 vma->size += next->size;
1530 nvkm_vmm_free_delete(vmm, next);
1531 }
1532
1533 nvkm_vmm_free_insert(vmm, vma);
1534}
1535
1536void
1537nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1538{
1539 const struct nvkm_vmm_page *page = vmm->func->page;
1540 struct nvkm_vma *next = vma;
1541
1542 BUG_ON(vma->part);
1543
1544 if (vma->mapref || !vma->sparse) {
1545 do {
1546 const bool mem = next->memory != NULL;
1547 const bool map = next->mapped;
1548 const u8 refd = next->refd;
1549 const u64 addr = next->addr;
1550 u64 size = next->size;
1551
1552
1553 while ((next = node(next, next)) && next->part &&
1554 (next->mapped == map) &&
1555 (next->memory != NULL) == mem &&
1556 (next->refd == refd))
1557 size += next->size;
1558
1559 if (map) {
1560
1561
1562
1563
1564 nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1565 size, vma->sparse,
1566 !mem);
1567 } else
1568 if (refd != NVKM_VMA_PAGE_NONE) {
1569
1570 nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1571 }
1572 } while (next && next->part);
1573 }
1574
1575
1576
1577
1578
1579 next = vma;
1580 do {
1581 if (next->mapped)
1582 nvkm_vmm_unmap_region(vmm, next);
1583 } while ((next = node(vma, next)) && next->part);
1584
1585 if (vma->sparse && !vma->mapref) {
1586
1587
1588
1589
1590
1591
1592
1593
1594 nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1595 } else
1596 if (vma->sparse) {
1597
1598
1599
1600
1601
1602
1603
1604
1605 nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1606 }
1607
1608
1609 nvkm_vmm_node_remove(vmm, vma);
1610
1611
1612 vma->page = NVKM_VMA_PAGE_NONE;
1613 vma->refd = NVKM_VMA_PAGE_NONE;
1614 vma->used = false;
1615 nvkm_vmm_put_region(vmm, vma);
1616}
1617
1618void
1619nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1620{
1621 struct nvkm_vma *vma = *pvma;
1622 if (vma) {
1623 mutex_lock(&vmm->mutex);
1624 nvkm_vmm_put_locked(vmm, vma);
1625 mutex_unlock(&vmm->mutex);
1626 *pvma = NULL;
1627 }
1628}
1629
1630int
1631nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1632 u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1633{
1634 const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1635 struct rb_node *node = NULL, *temp;
1636 struct nvkm_vma *vma = NULL, *tmp;
1637 u64 addr, tail;
1638 int ret;
1639
1640 VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1641 "shift: %d align: %d size: %016llx",
1642 getref, mapref, sparse, shift, align, size);
1643
1644
1645 if (unlikely(!size || (!getref && !mapref && sparse))) {
1646 VMM_DEBUG(vmm, "args %016llx %d %d %d",
1647 size, getref, mapref, sparse);
1648 return -EINVAL;
1649 }
1650
1651
1652
1653
1654
1655
1656
1657 if (unlikely((getref || vmm->func->page_block) && !shift)) {
1658 VMM_DEBUG(vmm, "page size required: %d %016llx",
1659 getref, vmm->func->page_block);
1660 return -EINVAL;
1661 }
1662
1663
1664
1665
1666 if (shift) {
1667 for (page = vmm->func->page; page->shift; page++) {
1668 if (shift == page->shift)
1669 break;
1670 }
1671
1672 if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1673 VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1674 return -EINVAL;
1675 }
1676 align = max_t(u8, align, shift);
1677 } else {
1678 align = max_t(u8, align, 12);
1679 }
1680
1681
1682 temp = vmm->free.rb_node;
1683 while (temp) {
1684 struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1685 if (this->size < size) {
1686 temp = temp->rb_right;
1687 } else {
1688 node = temp;
1689 temp = temp->rb_left;
1690 }
1691 }
1692
1693 if (unlikely(!node))
1694 return -ENOSPC;
1695
1696
1697
1698
1699 do {
1700 struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1701 struct nvkm_vma *prev = node(this, prev);
1702 struct nvkm_vma *next = node(this, next);
1703 const int p = page - vmm->func->page;
1704
1705 addr = this->addr;
1706 if (vmm->func->page_block && prev && prev->page != p)
1707 addr = ALIGN(addr, vmm->func->page_block);
1708 addr = ALIGN(addr, 1ULL << align);
1709
1710 tail = this->addr + this->size;
1711 if (vmm->func->page_block && next && next->page != p)
1712 tail = ALIGN_DOWN(tail, vmm->func->page_block);
1713
1714 if (addr <= tail && tail - addr >= size) {
1715 nvkm_vmm_free_remove(vmm, this);
1716 vma = this;
1717 break;
1718 }
1719 } while ((node = rb_next(node)));
1720
1721 if (unlikely(!vma))
1722 return -ENOSPC;
1723
1724
1725
1726
1727 if (addr != vma->addr) {
1728 if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1729 nvkm_vmm_put_region(vmm, vma);
1730 return -ENOMEM;
1731 }
1732 nvkm_vmm_free_insert(vmm, vma);
1733 vma = tmp;
1734 }
1735
1736 if (size != vma->size) {
1737 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1738 nvkm_vmm_put_region(vmm, vma);
1739 return -ENOMEM;
1740 }
1741 nvkm_vmm_free_insert(vmm, tmp);
1742 }
1743
1744
1745 if (sparse && getref)
1746 ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1747 else if (sparse)
1748 ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1749 else if (getref)
1750 ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1751 else
1752 ret = 0;
1753 if (ret) {
1754 nvkm_vmm_put_region(vmm, vma);
1755 return ret;
1756 }
1757
1758 vma->mapref = mapref && !getref;
1759 vma->sparse = sparse;
1760 vma->page = page - vmm->func->page;
1761 vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1762 vma->used = true;
1763 nvkm_vmm_node_insert(vmm, vma);
1764 *pvma = vma;
1765 return 0;
1766}
1767
1768int
1769nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1770{
1771 int ret;
1772 mutex_lock(&vmm->mutex);
1773 ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1774 mutex_unlock(&vmm->mutex);
1775 return ret;
1776}
1777
1778void
1779nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1780{
1781 if (inst && vmm && vmm->func->part) {
1782 mutex_lock(&vmm->mutex);
1783 vmm->func->part(vmm, inst);
1784 mutex_unlock(&vmm->mutex);
1785 }
1786}
1787
1788int
1789nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1790{
1791 int ret = 0;
1792 if (vmm->func->join) {
1793 mutex_lock(&vmm->mutex);
1794 ret = vmm->func->join(vmm, inst);
1795 mutex_unlock(&vmm->mutex);
1796 }
1797 return ret;
1798}
1799
1800static bool
1801nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
1802{
1803 const struct nvkm_vmm_desc *desc = it->desc;
1804 const int type = desc->type == SPT;
1805 nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1806 return false;
1807}
1808
1809int
1810nvkm_vmm_boot(struct nvkm_vmm *vmm)
1811{
1812 const struct nvkm_vmm_page *page = vmm->func->page;
1813 const u64 limit = vmm->limit - vmm->start;
1814 int ret;
1815
1816 while (page[1].shift)
1817 page++;
1818
1819 ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1820 if (ret)
1821 return ret;
1822
1823 nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
1824 nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1825 vmm->bootstrapped = true;
1826 return 0;
1827}
1828
1829static void
1830nvkm_vmm_del(struct kref *kref)
1831{
1832 struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1833 nvkm_vmm_dtor(vmm);
1834 kfree(vmm);
1835}
1836
1837void
1838nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1839{
1840 struct nvkm_vmm *vmm = *pvmm;
1841 if (vmm) {
1842 kref_put(&vmm->kref, nvkm_vmm_del);
1843 *pvmm = NULL;
1844 }
1845}
1846
1847struct nvkm_vmm *
1848nvkm_vmm_ref(struct nvkm_vmm *vmm)
1849{
1850 if (vmm)
1851 kref_get(&vmm->kref);
1852 return vmm;
1853}
1854
1855int
1856nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1857 u32 argc, struct lock_class_key *key, const char *name,
1858 struct nvkm_vmm **pvmm)
1859{
1860 struct nvkm_mmu *mmu = device->mmu;
1861 struct nvkm_vmm *vmm = NULL;
1862 int ret;
1863 ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc,
1864 key, name, &vmm);
1865 if (ret)
1866 nvkm_vmm_unref(&vmm);
1867 *pvmm = vmm;
1868 return ret;
1869}
1870