1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/dma-fence-array.h>
29#include <linux/interval_tree_generic.h>
30#include <linux/idr.h>
31
32#include <drm/amdgpu_drm.h>
33#include "amdgpu.h"
34#include "amdgpu_trace.h"
35#include "amdgpu_amdkfd.h"
36#include "amdgpu_gmc.h"
37#include "amdgpu_xgmi.h"
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60#define START(node) ((node)->start)
61#define LAST(node) ((node)->last)
62
63INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
64 START, LAST, static, amdgpu_vm_it)
65
66#undef START
67#undef LAST
68
69
70
71
72struct amdgpu_prt_cb {
73
74
75
76
77 struct amdgpu_device *adev;
78
79
80
81
82 struct dma_fence_cb cb;
83};
84
85
86
87
88
89
90
91
92
93
94static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
95 unsigned level)
96{
97 unsigned shift = 0xff;
98
99 switch (level) {
100 case AMDGPU_VM_PDB2:
101 case AMDGPU_VM_PDB1:
102 case AMDGPU_VM_PDB0:
103 shift = 9 * (AMDGPU_VM_PDB0 - level) +
104 adev->vm_manager.block_size;
105 break;
106 case AMDGPU_VM_PTB:
107 shift = 0;
108 break;
109 default:
110 dev_err(adev->dev, "the level%d isn't supported.\n", level);
111 }
112
113 return shift;
114}
115
116
117
118
119
120
121
122
123
124
125static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
126 unsigned level)
127{
128 unsigned shift = amdgpu_vm_level_shift(adev,
129 adev->vm_manager.root_level);
130
131 if (level == adev->vm_manager.root_level)
132
133 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift;
134 else if (level != AMDGPU_VM_PTB)
135
136 return 512;
137 else
138
139 return AMDGPU_VM_PTE_COUNT(adev);
140}
141
142
143
144
145
146
147
148
149
150static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev)
151{
152 unsigned shift;
153
154 shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level);
155 return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT);
156}
157
158
159
160
161
162
163
164
165
166
167static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev,
168 unsigned int level)
169{
170 if (level <= adev->vm_manager.root_level)
171 return 0xffffffff;
172 else if (level != AMDGPU_VM_PTB)
173 return 0x1ff;
174 else
175 return AMDGPU_VM_PTE_COUNT(adev) - 1;
176}
177
178
179
180
181
182
183
184
185
186
187static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
188{
189 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
190}
191
192
193
194
195
196
197
198
199
200static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
201{
202 struct amdgpu_vm *vm = vm_bo->vm;
203 struct amdgpu_bo *bo = vm_bo->bo;
204
205 vm_bo->moved = true;
206 if (bo->tbo.type == ttm_bo_type_kernel)
207 list_move(&vm_bo->vm_status, &vm->evicted);
208 else
209 list_move_tail(&vm_bo->vm_status, &vm->evicted);
210}
211
212
213
214
215
216
217
218
219static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
220{
221 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
222}
223
224
225
226
227
228
229
230
231
232static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
233{
234 list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
235}
236
237
238
239
240
241
242
243
244
245static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
246{
247 list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
248 vm_bo->moved = false;
249}
250
251
252
253
254
255
256
257
258
259static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
260{
261 spin_lock(&vm_bo->vm->invalidated_lock);
262 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
263 spin_unlock(&vm_bo->vm->invalidated_lock);
264}
265
266
267
268
269
270
271
272
273
274static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
275{
276 spin_lock(&vm_bo->vm->invalidated_lock);
277 list_del_init(&vm_bo->vm_status);
278 spin_unlock(&vm_bo->vm->invalidated_lock);
279}
280
281
282
283
284
285
286
287
288
289
290
291static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
292 struct amdgpu_vm *vm,
293 struct amdgpu_bo *bo)
294{
295 base->vm = vm;
296 base->bo = bo;
297 base->next = NULL;
298 INIT_LIST_HEAD(&base->vm_status);
299
300 if (!bo)
301 return;
302 base->next = bo->vm_bo;
303 bo->vm_bo = base;
304
305 if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
306 return;
307
308 vm->bulk_moveable = false;
309 if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
310 amdgpu_vm_bo_relocated(base);
311 else
312 amdgpu_vm_bo_idle(base);
313
314 if (bo->preferred_domains &
315 amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
316 return;
317
318
319
320
321
322
323 amdgpu_vm_bo_evicted(base);
324}
325
326
327
328
329
330
331
332
333
334static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt)
335{
336 struct amdgpu_bo *parent = pt->base.bo->parent;
337
338 if (!parent)
339 return NULL;
340
341 return container_of(parent->vm_bo, struct amdgpu_vm_pt, base);
342}
343
344
345
346
347struct amdgpu_vm_pt_cursor {
348 uint64_t pfn;
349 struct amdgpu_vm_pt *parent;
350 struct amdgpu_vm_pt *entry;
351 unsigned level;
352};
353
354
355
356
357
358
359
360
361
362
363
364static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
365 struct amdgpu_vm *vm, uint64_t start,
366 struct amdgpu_vm_pt_cursor *cursor)
367{
368 cursor->pfn = start;
369 cursor->parent = NULL;
370 cursor->entry = &vm->root;
371 cursor->level = adev->vm_manager.root_level;
372}
373
374
375
376
377
378
379
380
381
382
383
384static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
385 struct amdgpu_vm_pt_cursor *cursor)
386{
387 unsigned mask, shift, idx;
388
389 if (!cursor->entry->entries)
390 return false;
391
392 BUG_ON(!cursor->entry->base.bo);
393 mask = amdgpu_vm_entries_mask(adev, cursor->level);
394 shift = amdgpu_vm_level_shift(adev, cursor->level);
395
396 ++cursor->level;
397 idx = (cursor->pfn >> shift) & mask;
398 cursor->parent = cursor->entry;
399 cursor->entry = &cursor->entry->entries[idx];
400 return true;
401}
402
403
404
405
406
407
408
409
410
411
412
413static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
414 struct amdgpu_vm_pt_cursor *cursor)
415{
416 unsigned shift, num_entries;
417
418
419 if (!cursor->parent)
420 return false;
421
422
423 shift = amdgpu_vm_level_shift(adev, cursor->level - 1);
424 num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);
425
426 if (cursor->entry == &cursor->parent->entries[num_entries - 1])
427 return false;
428
429 cursor->pfn += 1ULL << shift;
430 cursor->pfn &= ~((1ULL << shift) - 1);
431 ++cursor->entry;
432 return true;
433}
434
435
436
437
438
439
440
441
442
443
444static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
445{
446 if (!cursor->parent)
447 return false;
448
449 --cursor->level;
450 cursor->entry = cursor->parent;
451 cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
452 return true;
453}
454
455
456
457
458
459
460
461
462
463static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
464 struct amdgpu_vm_pt_cursor *cursor)
465{
466
467 if (amdgpu_vm_pt_descendant(adev, cursor))
468 return;
469
470
471 while (!amdgpu_vm_pt_sibling(adev, cursor)) {
472
473 if (!amdgpu_vm_pt_ancestor(cursor)) {
474 cursor->pfn = ~0ll;
475 return;
476 }
477 }
478}
479
480
481
482
483
484
485
486
487
488
489static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
490 struct amdgpu_vm *vm,
491 struct amdgpu_vm_pt_cursor *start,
492 struct amdgpu_vm_pt_cursor *cursor)
493{
494 if (start)
495 *cursor = *start;
496 else
497 amdgpu_vm_pt_start(adev, vm, 0, cursor);
498 while (amdgpu_vm_pt_descendant(adev, cursor));
499}
500
501
502
503
504
505
506
507
508
509
510static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
511 struct amdgpu_vm_pt *entry)
512{
513 return entry && (!start || entry != start->entry);
514}
515
516
517
518
519
520
521
522
523
524static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
525 struct amdgpu_vm_pt_cursor *cursor)
526{
527 if (!cursor->entry)
528 return;
529
530 if (!cursor->parent)
531 cursor->entry = NULL;
532 else if (amdgpu_vm_pt_sibling(adev, cursor))
533 while (amdgpu_vm_pt_descendant(adev, cursor));
534 else
535 amdgpu_vm_pt_ancestor(cursor);
536}
537
538
539
540
541#define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \
542 for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \
543 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
544 amdgpu_vm_pt_continue_dfs((start), (entry)); \
545 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
546
547
548
549
550
551
552
553
554
555
556
557void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
558 struct list_head *validated,
559 struct amdgpu_bo_list_entry *entry)
560{
561 entry->priority = 0;
562 entry->tv.bo = &vm->root.base.bo->tbo;
563
564 entry->tv.num_shared = 3;
565 entry->user_pages = NULL;
566 list_add(&entry->tv.head, validated);
567}
568
569void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo)
570{
571 struct amdgpu_bo *abo;
572 struct amdgpu_vm_bo_base *bo_base;
573
574 if (!amdgpu_bo_is_amdgpu_bo(bo))
575 return;
576
577 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT)
578 return;
579
580 abo = ttm_to_amdgpu_bo(bo);
581 if (!abo->parent)
582 return;
583 for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) {
584 struct amdgpu_vm *vm = bo_base->vm;
585
586 if (abo->tbo.resv == vm->root.base.bo->tbo.resv)
587 vm->bulk_moveable = false;
588 }
589
590}
591
592
593
594
595
596
597
598
599
600void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
601 struct amdgpu_vm *vm)
602{
603 struct ttm_bo_global *glob = adev->mman.bdev.glob;
604 struct amdgpu_vm_bo_base *bo_base;
605
606#if 0
607 if (vm->bulk_moveable) {
608 spin_lock(&glob->lru_lock);
609 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);
610 spin_unlock(&glob->lru_lock);
611 return;
612 }
613#endif
614
615 memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));
616
617 spin_lock(&glob->lru_lock);
618 list_for_each_entry(bo_base, &vm->idle, vm_status) {
619 struct amdgpu_bo *bo = bo_base->bo;
620
621 if (!bo->parent)
622 continue;
623
624 ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move);
625 if (bo->shadow)
626 ttm_bo_move_to_lru_tail(&bo->shadow->tbo,
627 &vm->lru_bulk_move);
628 }
629 spin_unlock(&glob->lru_lock);
630
631 vm->bulk_moveable = true;
632}
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
648 int (*validate)(void *p, struct amdgpu_bo *bo),
649 void *param)
650{
651 struct amdgpu_vm_bo_base *bo_base, *tmp;
652 int r = 0;
653
654 vm->bulk_moveable &= list_empty(&vm->evicted);
655
656 list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
657 struct amdgpu_bo *bo = bo_base->bo;
658
659 r = validate(param, bo);
660 if (r)
661 break;
662
663 if (bo->tbo.type != ttm_bo_type_kernel) {
664 amdgpu_vm_bo_moved(bo_base);
665 } else {
666 vm->update_funcs->map_table(bo);
667 if (bo->parent)
668 amdgpu_vm_bo_relocated(bo_base);
669 else
670 amdgpu_vm_bo_idle(bo_base);
671 }
672 }
673
674 return r;
675}
676
677
678
679
680
681
682
683
684
685
686
687bool amdgpu_vm_ready(struct amdgpu_vm *vm)
688{
689 return list_empty(&vm->evicted);
690}
691
692
693
694
695
696
697
698
699
700
701
702
703
704static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
705 struct amdgpu_vm *vm,
706 struct amdgpu_bo *bo)
707{
708 struct ttm_operation_ctx ctx = { true, false };
709 unsigned level = adev->vm_manager.root_level;
710 struct amdgpu_vm_update_params params;
711 struct amdgpu_bo *ancestor = bo;
712 unsigned entries, ats_entries;
713 uint64_t addr;
714 int r;
715
716
717 if (ancestor->parent) {
718 ++level;
719 while (ancestor->parent->parent) {
720 ++level;
721 ancestor = ancestor->parent;
722 }
723 }
724
725 entries = amdgpu_bo_size(bo) / 8;
726 if (!vm->pte_support_ats) {
727 ats_entries = 0;
728
729 } else if (!bo->parent) {
730 ats_entries = amdgpu_vm_num_ats_entries(adev);
731 ats_entries = min(ats_entries, entries);
732 entries -= ats_entries;
733
734 } else {
735 struct amdgpu_vm_pt *pt;
736
737 pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base);
738 ats_entries = amdgpu_vm_num_ats_entries(adev);
739 if ((pt - vm->root.entries) >= ats_entries) {
740 ats_entries = 0;
741 } else {
742 ats_entries = entries;
743 entries = 0;
744 }
745 }
746
747 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
748 if (r)
749 return r;
750
751 if (bo->shadow) {
752 r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement,
753 &ctx);
754 if (r)
755 return r;
756 }
757
758 r = vm->update_funcs->map_table(bo);
759 if (r)
760 return r;
761
762 memset(¶ms, 0, sizeof(params));
763 params.adev = adev;
764 params.vm = vm;
765
766 r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_KFD, NULL);
767 if (r)
768 return r;
769
770 addr = 0;
771 if (ats_entries) {
772 uint64_t value = 0, flags;
773
774 flags = AMDGPU_PTE_DEFAULT_ATC;
775 if (level != AMDGPU_VM_PTB) {
776
777 flags |= AMDGPU_PDE_PTE;
778 amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
779 }
780
781 r = vm->update_funcs->update(¶ms, bo, addr, 0, ats_entries,
782 value, flags);
783 if (r)
784 return r;
785
786 addr += ats_entries * 8;
787 }
788
789 if (entries) {
790 uint64_t value = 0, flags = 0;
791
792 if (adev->asic_type >= CHIP_VEGA10) {
793 if (level != AMDGPU_VM_PTB) {
794
795 flags |= AMDGPU_PDE_PTE;
796 amdgpu_gmc_get_vm_pde(adev, level,
797 &value, &flags);
798 } else {
799
800 flags = AMDGPU_PTE_EXECUTABLE;
801 }
802 }
803
804 r = vm->update_funcs->update(¶ms, bo, addr, 0, entries,
805 value, flags);
806 if (r)
807 return r;
808 }
809
810 return vm->update_funcs->commit(¶ms, NULL);
811}
812
813
814
815
816
817
818
819
820static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm,
821 int level, struct amdgpu_bo_param *bp)
822{
823 memset(bp, 0, sizeof(*bp));
824
825 bp->size = amdgpu_vm_bo_size(adev, level);
826 bp->byte_align = AMDGPU_GPU_PAGE_SIZE;
827 bp->domain = AMDGPU_GEM_DOMAIN_VRAM;
828 bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain);
829 bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
830 AMDGPU_GEM_CREATE_CPU_GTT_USWC;
831 if (vm->use_cpu_for_update)
832 bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
833 else if (!vm->root.base.bo || vm->root.base.bo->shadow)
834 bp->flags |= AMDGPU_GEM_CREATE_SHADOW;
835 bp->type = ttm_bo_type_kernel;
836 if (vm->root.base.bo)
837 bp->resv = vm->root.base.bo->tbo.resv;
838}
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
854 struct amdgpu_vm *vm,
855 struct amdgpu_vm_pt_cursor *cursor)
856{
857 struct amdgpu_vm_pt *entry = cursor->entry;
858 struct amdgpu_bo_param bp;
859 struct amdgpu_bo *pt;
860 int r;
861
862 if (cursor->level < AMDGPU_VM_PTB && !entry->entries) {
863 unsigned num_entries;
864
865 num_entries = amdgpu_vm_num_entries(adev, cursor->level);
866 entry->entries = kvmalloc_array(num_entries,
867 sizeof(*entry->entries),
868 GFP_KERNEL | __GFP_ZERO);
869 if (!entry->entries)
870 return -ENOMEM;
871 }
872
873 if (entry->base.bo)
874 return 0;
875
876 amdgpu_vm_bo_param(adev, vm, cursor->level, &bp);
877
878 r = amdgpu_bo_create(adev, &bp, &pt);
879 if (r)
880 return r;
881
882
883
884
885 pt->parent = amdgpu_bo_ref(cursor->parent->base.bo);
886 amdgpu_vm_bo_base_init(&entry->base, vm, pt);
887
888 r = amdgpu_vm_clear_bo(adev, vm, pt);
889 if (r)
890 goto error_free_pt;
891
892 return 0;
893
894error_free_pt:
895 amdgpu_bo_unref(&pt->shadow);
896 amdgpu_bo_unref(&pt);
897 return r;
898}
899
900
901
902
903
904
905static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry)
906{
907 if (entry->base.bo) {
908 entry->base.bo->vm_bo = NULL;
909 list_del(&entry->base.vm_status);
910 amdgpu_bo_unref(&entry->base.bo->shadow);
911 amdgpu_bo_unref(&entry->base.bo);
912 }
913 kvfree(entry->entries);
914 entry->entries = NULL;
915}
916
917
918
919
920
921
922
923
924
925
926static void amdgpu_vm_free_pts(struct amdgpu_device *adev,
927 struct amdgpu_vm *vm,
928 struct amdgpu_vm_pt_cursor *start)
929{
930 struct amdgpu_vm_pt_cursor cursor;
931 struct amdgpu_vm_pt *entry;
932
933 vm->bulk_moveable = false;
934
935 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
936 amdgpu_vm_free_table(entry);
937
938 if (start)
939 amdgpu_vm_free_table(start->entry);
940}
941
942
943
944
945
946
947void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
948{
949 const struct amdgpu_ip_block *ip_block;
950 bool has_compute_vm_bug;
951 struct amdgpu_ring *ring;
952 int i;
953
954 has_compute_vm_bug = false;
955
956 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
957 if (ip_block) {
958
959
960 if (ip_block->version->major <= 7)
961 has_compute_vm_bug = true;
962 else if (ip_block->version->major == 8)
963 if (adev->gfx.mec_fw_version < 673)
964 has_compute_vm_bug = true;
965 }
966
967 for (i = 0; i < adev->num_rings; i++) {
968 ring = adev->rings[i];
969 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
970
971 ring->has_compute_vm_bug = has_compute_vm_bug;
972 else
973 ring->has_compute_vm_bug = false;
974 }
975}
976
977
978
979
980
981
982
983
984
985
986bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
987 struct amdgpu_job *job)
988{
989 struct amdgpu_device *adev = ring->adev;
990 unsigned vmhub = ring->funcs->vmhub;
991 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
992 struct amdgpu_vmid *id;
993 bool gds_switch_needed;
994 bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
995
996 if (job->vmid == 0)
997 return false;
998 id = &id_mgr->ids[job->vmid];
999 gds_switch_needed = ring->funcs->emit_gds_switch && (
1000 id->gds_base != job->gds_base ||
1001 id->gds_size != job->gds_size ||
1002 id->gws_base != job->gws_base ||
1003 id->gws_size != job->gws_size ||
1004 id->oa_base != job->oa_base ||
1005 id->oa_size != job->oa_size);
1006
1007 if (amdgpu_vmid_had_gpu_reset(adev, id))
1008 return true;
1009
1010 return vm_flush_needed || gds_switch_needed;
1011}
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
1026{
1027 struct amdgpu_device *adev = ring->adev;
1028 unsigned vmhub = ring->funcs->vmhub;
1029 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
1030 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
1031 bool gds_switch_needed = ring->funcs->emit_gds_switch && (
1032 id->gds_base != job->gds_base ||
1033 id->gds_size != job->gds_size ||
1034 id->gws_base != job->gws_base ||
1035 id->gws_size != job->gws_size ||
1036 id->oa_base != job->oa_base ||
1037 id->oa_size != job->oa_size);
1038 bool vm_flush_needed = job->vm_needs_flush;
1039 bool pasid_mapping_needed = id->pasid != job->pasid ||
1040 !id->pasid_mapping ||
1041 !dma_fence_is_signaled(id->pasid_mapping);
1042 struct dma_fence *fence = NULL;
1043 unsigned patch_offset = 0;
1044 int r;
1045
1046 if (amdgpu_vmid_had_gpu_reset(adev, id)) {
1047 gds_switch_needed = true;
1048 vm_flush_needed = true;
1049 pasid_mapping_needed = true;
1050 }
1051
1052 gds_switch_needed &= !!ring->funcs->emit_gds_switch;
1053 vm_flush_needed &= !!ring->funcs->emit_vm_flush &&
1054 job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
1055 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
1056 ring->funcs->emit_wreg;
1057
1058 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
1059 return 0;
1060
1061 if (ring->funcs->init_cond_exec)
1062 patch_offset = amdgpu_ring_init_cond_exec(ring);
1063
1064 if (need_pipe_sync)
1065 amdgpu_ring_emit_pipeline_sync(ring);
1066
1067 if (vm_flush_needed) {
1068 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
1069 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
1070 }
1071
1072 if (pasid_mapping_needed)
1073 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
1074
1075 if (vm_flush_needed || pasid_mapping_needed) {
1076 r = amdgpu_fence_emit(ring, &fence, 0);
1077 if (r)
1078 return r;
1079 }
1080
1081 if (vm_flush_needed) {
1082 mutex_lock(&id_mgr->lock);
1083 dma_fence_put(id->last_flush);
1084 id->last_flush = dma_fence_get(fence);
1085 id->current_gpu_reset_count =
1086 atomic_read(&adev->gpu_reset_counter);
1087 mutex_unlock(&id_mgr->lock);
1088 }
1089
1090 if (pasid_mapping_needed) {
1091 id->pasid = job->pasid;
1092 dma_fence_put(id->pasid_mapping);
1093 id->pasid_mapping = dma_fence_get(fence);
1094 }
1095 dma_fence_put(fence);
1096
1097 if (ring->funcs->emit_gds_switch && gds_switch_needed) {
1098 id->gds_base = job->gds_base;
1099 id->gds_size = job->gds_size;
1100 id->gws_base = job->gws_base;
1101 id->gws_size = job->gws_size;
1102 id->oa_base = job->oa_base;
1103 id->oa_size = job->oa_size;
1104 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
1105 job->gds_size, job->gws_base,
1106 job->gws_size, job->oa_base,
1107 job->oa_size);
1108 }
1109
1110 if (ring->funcs->patch_cond_exec)
1111 amdgpu_ring_patch_cond_exec(ring, patch_offset);
1112
1113
1114 if (ring->funcs->emit_switch_buffer) {
1115 amdgpu_ring_emit_switch_buffer(ring);
1116 amdgpu_ring_emit_switch_buffer(ring);
1117 }
1118 return 0;
1119}
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
1137 struct amdgpu_bo *bo)
1138{
1139 struct amdgpu_vm_bo_base *base;
1140
1141 for (base = bo->vm_bo; base; base = base->next) {
1142 if (base->vm != vm)
1143 continue;
1144
1145 return container_of(base, struct amdgpu_bo_va, base);
1146 }
1147 return NULL;
1148}
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
1163{
1164 uint64_t result;
1165
1166
1167 result = pages_addr[addr >> PAGE_SHIFT];
1168
1169
1170 result |= addr & (~PAGE_MASK);
1171
1172 result &= 0xFFFFFFFFFFFFF000ULL;
1173
1174 return result;
1175}
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params,
1187 struct amdgpu_vm *vm,
1188 struct amdgpu_vm_pt *entry)
1189{
1190 struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry);
1191 struct amdgpu_bo *bo = parent->base.bo, *pbo;
1192 uint64_t pde, pt, flags;
1193 unsigned level;
1194
1195 for (level = 0, pbo = bo->parent; pbo; ++level)
1196 pbo = pbo->parent;
1197
1198 level += params->adev->vm_manager.root_level;
1199 amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
1200 pde = (entry - parent->entries) * 8;
1201 return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags);
1202}
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev,
1213 struct amdgpu_vm *vm)
1214{
1215 struct amdgpu_vm_pt_cursor cursor;
1216 struct amdgpu_vm_pt *entry;
1217
1218 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)
1219 if (entry->base.bo && !entry->base.moved)
1220 amdgpu_vm_bo_relocated(&entry->base);
1221}
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234int amdgpu_vm_update_directories(struct amdgpu_device *adev,
1235 struct amdgpu_vm *vm)
1236{
1237 struct amdgpu_vm_update_params params;
1238 int r;
1239
1240 if (list_empty(&vm->relocated))
1241 return 0;
1242
1243 memset(¶ms, 0, sizeof(params));
1244 params.adev = adev;
1245 params.vm = vm;
1246
1247 r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_VM, NULL);
1248 if (r)
1249 return r;
1250
1251 while (!list_empty(&vm->relocated)) {
1252 struct amdgpu_vm_pt *entry;
1253
1254 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt,
1255 base.vm_status);
1256 amdgpu_vm_bo_idle(&entry->base);
1257
1258 r = amdgpu_vm_update_pde(¶ms, vm, entry);
1259 if (r)
1260 goto error;
1261 }
1262
1263 r = vm->update_funcs->commit(¶ms, &vm->last_update);
1264 if (r)
1265 goto error;
1266 return 0;
1267
1268error:
1269 amdgpu_vm_invalidate_pds(adev, vm);
1270 return r;
1271}
1272
1273
1274
1275
1276
1277
1278static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params,
1279 struct amdgpu_bo *bo, unsigned level,
1280 uint64_t pe, uint64_t addr,
1281 unsigned count, uint32_t incr,
1282 uint64_t flags)
1283
1284{
1285 if (level != AMDGPU_VM_PTB) {
1286 flags |= AMDGPU_PDE_PTE;
1287 amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags);
1288
1289 } else if (params->adev->asic_type >= CHIP_VEGA10 &&
1290 !(flags & AMDGPU_PTE_VALID) &&
1291 !(flags & AMDGPU_PTE_PRT)) {
1292
1293
1294 flags |= AMDGPU_PTE_EXECUTABLE;
1295 }
1296
1297 params->vm->update_funcs->update(params, bo, pe, addr, count, incr,
1298 flags);
1299}
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params,
1314 uint64_t start, uint64_t end, uint64_t flags,
1315 unsigned int *frag, uint64_t *frag_end)
1316{
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338 unsigned max_frag;
1339
1340 if (params->adev->asic_type < CHIP_VEGA10)
1341 max_frag = params->adev->vm_manager.fragment_size;
1342 else
1343 max_frag = 31;
1344
1345
1346 if (params->pages_addr) {
1347 *frag = 0;
1348 *frag_end = end;
1349 return;
1350 }
1351
1352
1353 *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1);
1354 if (*frag >= max_frag) {
1355 *frag = max_frag;
1356 *frag_end = end & ~((1ULL << max_frag) - 1);
1357 } else {
1358 *frag_end = start + (1 << *frag);
1359 }
1360}
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
1377 uint64_t start, uint64_t end,
1378 uint64_t dst, uint64_t flags)
1379{
1380 struct amdgpu_device *adev = params->adev;
1381 struct amdgpu_vm_pt_cursor cursor;
1382 uint64_t frag_start = start, frag_end;
1383 unsigned int frag;
1384 int r;
1385
1386
1387 amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end);
1388
1389
1390 amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
1391 while (cursor.pfn < end) {
1392 unsigned shift, parent_shift, mask;
1393 uint64_t incr, entry_end, pe_start;
1394 struct amdgpu_bo *pt;
1395
1396 r = amdgpu_vm_alloc_pts(params->adev, params->vm, &cursor);
1397 if (r)
1398 return r;
1399
1400 pt = cursor.entry->base.bo;
1401
1402
1403 if (cursor.level == adev->vm_manager.root_level) {
1404 if (!amdgpu_vm_pt_descendant(adev, &cursor))
1405 return -ENOENT;
1406 continue;
1407 }
1408
1409 shift = amdgpu_vm_level_shift(adev, cursor.level);
1410 parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
1411 if (adev->asic_type < CHIP_VEGA10 &&
1412 (flags & AMDGPU_PTE_VALID)) {
1413
1414 if (cursor.level != AMDGPU_VM_PTB) {
1415 if (!amdgpu_vm_pt_descendant(adev, &cursor))
1416 return -ENOENT;
1417 continue;
1418 }
1419 } else if (frag < shift) {
1420
1421
1422
1423
1424 if (!amdgpu_vm_pt_descendant(adev, &cursor))
1425 return -ENOENT;
1426 continue;
1427 } else if (frag >= parent_shift &&
1428 cursor.level - 1 != adev->vm_manager.root_level) {
1429
1430
1431
1432
1433 if (!amdgpu_vm_pt_ancestor(&cursor))
1434 return -ENOENT;
1435 continue;
1436 }
1437
1438
1439 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift;
1440 mask = amdgpu_vm_entries_mask(adev, cursor.level);
1441 pe_start = ((cursor.pfn >> shift) & mask) * 8;
1442 entry_end = (uint64_t)(mask + 1) << shift;
1443 entry_end += cursor.pfn & ~(entry_end - 1);
1444 entry_end = min(entry_end, end);
1445
1446 do {
1447 uint64_t upd_end = min(entry_end, frag_end);
1448 unsigned nptes = (upd_end - frag_start) >> shift;
1449
1450 amdgpu_vm_update_flags(params, pt, cursor.level,
1451 pe_start, dst, nptes, incr,
1452 flags | AMDGPU_PTE_FRAG(frag));
1453
1454 pe_start += nptes * 8;
1455 dst += (uint64_t)nptes * AMDGPU_GPU_PAGE_SIZE << shift;
1456
1457 frag_start = upd_end;
1458 if (frag_start >= frag_end) {
1459
1460 amdgpu_vm_fragment(params, frag_start, end,
1461 flags, &frag, &frag_end);
1462 if (frag < shift)
1463 break;
1464 }
1465 } while (frag_start < entry_end);
1466
1467 if (amdgpu_vm_pt_descendant(adev, &cursor)) {
1468
1469 while (cursor.pfn < frag_start) {
1470 amdgpu_vm_free_pts(adev, params->vm, &cursor);
1471 amdgpu_vm_pt_next(adev, &cursor);
1472 }
1473
1474 } else if (frag >= shift) {
1475
1476 amdgpu_vm_pt_next(adev, &cursor);
1477 }
1478 }
1479
1480 return 0;
1481}
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1502 struct dma_fence *exclusive,
1503 dma_addr_t *pages_addr,
1504 struct amdgpu_vm *vm,
1505 uint64_t start, uint64_t last,
1506 uint64_t flags, uint64_t addr,
1507 struct dma_fence **fence)
1508{
1509 struct amdgpu_vm_update_params params;
1510 void *owner = AMDGPU_FENCE_OWNER_VM;
1511 int r;
1512
1513 memset(¶ms, 0, sizeof(params));
1514 params.adev = adev;
1515 params.vm = vm;
1516 params.pages_addr = pages_addr;
1517
1518
1519 if (!(flags & AMDGPU_PTE_VALID))
1520 owner = AMDGPU_FENCE_OWNER_KFD;
1521
1522 r = vm->update_funcs->prepare(¶ms, owner, exclusive);
1523 if (r)
1524 return r;
1525
1526 r = amdgpu_vm_update_ptes(¶ms, start, last + 1, addr, flags);
1527 if (r)
1528 return r;
1529
1530 return vm->update_funcs->commit(¶ms, fence);
1531}
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1553 struct dma_fence *exclusive,
1554 dma_addr_t *pages_addr,
1555 struct amdgpu_vm *vm,
1556 struct amdgpu_bo_va_mapping *mapping,
1557 uint64_t flags,
1558 struct amdgpu_device *bo_adev,
1559 struct drm_mm_node *nodes,
1560 struct dma_fence **fence)
1561{
1562 unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
1563 uint64_t pfn, start = mapping->start;
1564 int r;
1565
1566
1567
1568
1569 if (!(mapping->flags & AMDGPU_PTE_READABLE))
1570 flags &= ~AMDGPU_PTE_READABLE;
1571 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1572 flags &= ~AMDGPU_PTE_WRITEABLE;
1573
1574 flags &= ~AMDGPU_PTE_EXECUTABLE;
1575 flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1576
1577 if (adev->asic_type == CHIP_NAVI10) {
1578 flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK;
1579 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK);
1580 } else {
1581 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1582 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK);
1583 }
1584
1585 if ((mapping->flags & AMDGPU_PTE_PRT) &&
1586 (adev->asic_type >= CHIP_VEGA10)) {
1587 flags |= AMDGPU_PTE_PRT;
1588 if (adev->asic_type >= CHIP_NAVI10) {
1589 flags |= AMDGPU_PTE_SNOOPED;
1590 flags |= AMDGPU_PTE_LOG;
1591 flags |= AMDGPU_PTE_SYSTEM;
1592 }
1593 flags &= ~AMDGPU_PTE_VALID;
1594 }
1595
1596 trace_amdgpu_vm_bo_update(mapping);
1597
1598 pfn = mapping->offset >> PAGE_SHIFT;
1599 if (nodes) {
1600 while (pfn >= nodes->size) {
1601 pfn -= nodes->size;
1602 ++nodes;
1603 }
1604 }
1605
1606 do {
1607 dma_addr_t *dma_addr = NULL;
1608 uint64_t max_entries;
1609 uint64_t addr, last;
1610
1611 if (nodes) {
1612 addr = nodes->start << PAGE_SHIFT;
1613 max_entries = (nodes->size - pfn) *
1614 AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1615 } else {
1616 addr = 0;
1617 max_entries = S64_MAX;
1618 }
1619
1620 if (pages_addr) {
1621 uint64_t count;
1622
1623 for (count = 1;
1624 count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1625 ++count) {
1626 uint64_t idx = pfn + count;
1627
1628 if (pages_addr[idx] !=
1629 (pages_addr[idx - 1] + PAGE_SIZE))
1630 break;
1631 }
1632
1633 if (count < min_linear_pages) {
1634 addr = pfn << PAGE_SHIFT;
1635 dma_addr = pages_addr;
1636 } else {
1637 addr = pages_addr[pfn];
1638 max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1639 }
1640
1641 } else if (flags & AMDGPU_PTE_VALID) {
1642 addr += bo_adev->vm_manager.vram_base_offset;
1643 addr += pfn << PAGE_SHIFT;
1644 }
1645
1646 last = min((uint64_t)mapping->last, start + max_entries - 1);
1647 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
1648 start, last, flags, addr,
1649 fence);
1650 if (r)
1651 return r;
1652
1653 pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1654 if (nodes && nodes->size == pfn) {
1655 pfn = 0;
1656 ++nodes;
1657 }
1658 start = last + 1;
1659
1660 } while (unlikely(start != mapping->last + 1));
1661
1662 return 0;
1663}
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1678 struct amdgpu_bo_va *bo_va,
1679 bool clear)
1680{
1681 struct amdgpu_bo *bo = bo_va->base.bo;
1682 struct amdgpu_vm *vm = bo_va->base.vm;
1683 struct amdgpu_bo_va_mapping *mapping;
1684 dma_addr_t *pages_addr = NULL;
1685 struct ttm_mem_reg *mem;
1686 struct drm_mm_node *nodes;
1687 struct dma_fence *exclusive, **last_update;
1688 uint64_t flags;
1689 struct amdgpu_device *bo_adev = adev;
1690 int r;
1691
1692 if (clear || !bo) {
1693 mem = NULL;
1694 nodes = NULL;
1695 exclusive = NULL;
1696 } else {
1697 struct ttm_dma_tt *ttm;
1698
1699 mem = &bo->tbo.mem;
1700 nodes = mem->mm_node;
1701 if (mem->mem_type == TTM_PL_TT) {
1702 ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm);
1703 pages_addr = ttm->dma_address;
1704 }
1705 exclusive = reservation_object_get_excl(bo->tbo.resv);
1706 }
1707
1708 if (bo) {
1709 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1710 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1711 } else {
1712 flags = 0x0;
1713 }
1714
1715 if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
1716 last_update = &vm->last_update;
1717 else
1718 last_update = &bo_va->last_pt_update;
1719
1720 if (!clear && bo_va->base.moved) {
1721 bo_va->base.moved = false;
1722 list_splice_init(&bo_va->valids, &bo_va->invalids);
1723
1724 } else if (bo_va->cleared != clear) {
1725 list_splice_init(&bo_va->valids, &bo_va->invalids);
1726 }
1727
1728 list_for_each_entry(mapping, &bo_va->invalids, list) {
1729 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
1730 mapping, flags, bo_adev, nodes,
1731 last_update);
1732 if (r)
1733 return r;
1734 }
1735
1736 if (vm->use_cpu_for_update) {
1737
1738 mb();
1739 amdgpu_asic_flush_hdp(adev, NULL);
1740 }
1741
1742
1743
1744
1745
1746 if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
1747 uint32_t mem_type = bo->tbo.mem.mem_type;
1748
1749 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
1750 amdgpu_vm_bo_evicted(&bo_va->base);
1751 else
1752 amdgpu_vm_bo_idle(&bo_va->base);
1753 } else {
1754 amdgpu_vm_bo_done(&bo_va->base);
1755 }
1756
1757 list_splice_init(&bo_va->invalids, &bo_va->valids);
1758 bo_va->cleared = clear;
1759
1760 if (trace_amdgpu_vm_bo_mapping_enabled()) {
1761 list_for_each_entry(mapping, &bo_va->valids, list)
1762 trace_amdgpu_vm_bo_mapping(mapping);
1763 }
1764
1765 return 0;
1766}
1767
1768
1769
1770
1771
1772
1773static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1774{
1775 unsigned long flags;
1776 bool enable;
1777
1778 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1779 enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1780 adev->gmc.gmc_funcs->set_prt(adev, enable);
1781 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1782}
1783
1784
1785
1786
1787
1788
1789static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1790{
1791 if (!adev->gmc.gmc_funcs->set_prt)
1792 return;
1793
1794 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1795 amdgpu_vm_update_prt_state(adev);
1796}
1797
1798
1799
1800
1801
1802
1803static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1804{
1805 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1806 amdgpu_vm_update_prt_state(adev);
1807}
1808
1809
1810
1811
1812
1813
1814
1815static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1816{
1817 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1818
1819 amdgpu_vm_prt_put(cb->adev);
1820 kfree(cb);
1821}
1822
1823
1824
1825
1826
1827
1828
1829static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1830 struct dma_fence *fence)
1831{
1832 struct amdgpu_prt_cb *cb;
1833
1834 if (!adev->gmc.gmc_funcs->set_prt)
1835 return;
1836
1837 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1838 if (!cb) {
1839
1840 if (fence)
1841 dma_fence_wait(fence, false);
1842
1843 amdgpu_vm_prt_put(adev);
1844 } else {
1845 cb->adev = adev;
1846 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1847 amdgpu_vm_prt_cb))
1848 amdgpu_vm_prt_cb(fence, &cb->cb);
1849 }
1850}
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1863 struct amdgpu_vm *vm,
1864 struct amdgpu_bo_va_mapping *mapping,
1865 struct dma_fence *fence)
1866{
1867 if (mapping->flags & AMDGPU_PTE_PRT)
1868 amdgpu_vm_add_prt_cb(adev, fence);
1869 kfree(mapping);
1870}
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1881{
1882 struct reservation_object *resv = vm->root.base.bo->tbo.resv;
1883 struct dma_fence *excl, **shared;
1884 unsigned i, shared_count;
1885 int r;
1886
1887 r = reservation_object_get_fences_rcu(resv, &excl,
1888 &shared_count, &shared);
1889 if (r) {
1890
1891
1892
1893 reservation_object_wait_timeout_rcu(resv, true, false,
1894 MAX_SCHEDULE_TIMEOUT);
1895 return;
1896 }
1897
1898
1899 amdgpu_vm_prt_get(adev);
1900 amdgpu_vm_add_prt_cb(adev, excl);
1901
1902 for (i = 0; i < shared_count; ++i) {
1903 amdgpu_vm_prt_get(adev);
1904 amdgpu_vm_add_prt_cb(adev, shared[i]);
1905 }
1906
1907 kfree(shared);
1908}
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1926 struct amdgpu_vm *vm,
1927 struct dma_fence **fence)
1928{
1929 struct amdgpu_bo_va_mapping *mapping;
1930 uint64_t init_pte_value = 0;
1931 struct dma_fence *f = NULL;
1932 int r;
1933
1934 while (!list_empty(&vm->freed)) {
1935 mapping = list_first_entry(&vm->freed,
1936 struct amdgpu_bo_va_mapping, list);
1937 list_del(&mapping->list);
1938
1939 if (vm->pte_support_ats &&
1940 mapping->start < AMDGPU_GMC_HOLE_START)
1941 init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1942
1943 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
1944 mapping->start, mapping->last,
1945 init_pte_value, 0, &f);
1946 amdgpu_vm_free_mapping(adev, vm, mapping, f);
1947 if (r) {
1948 dma_fence_put(f);
1949 return r;
1950 }
1951 }
1952
1953 if (fence && f) {
1954 dma_fence_put(*fence);
1955 *fence = f;
1956 } else {
1957 dma_fence_put(f);
1958 }
1959
1960 return 0;
1961
1962}
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1978 struct amdgpu_vm *vm)
1979{
1980 struct amdgpu_bo_va *bo_va, *tmp;
1981 struct reservation_object *resv;
1982 bool clear;
1983 int r;
1984
1985 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
1986
1987 r = amdgpu_vm_bo_update(adev, bo_va, false);
1988 if (r)
1989 return r;
1990 }
1991
1992 spin_lock(&vm->invalidated_lock);
1993 while (!list_empty(&vm->invalidated)) {
1994 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1995 base.vm_status);
1996 resv = bo_va->base.bo->tbo.resv;
1997 spin_unlock(&vm->invalidated_lock);
1998
1999
2000 if (!amdgpu_vm_debug && reservation_object_trylock(resv))
2001 clear = false;
2002
2003 else
2004 clear = true;
2005
2006 r = amdgpu_vm_bo_update(adev, bo_va, clear);
2007 if (r)
2008 return r;
2009
2010 if (!clear)
2011 reservation_object_unlock(resv);
2012 spin_lock(&vm->invalidated_lock);
2013 }
2014 spin_unlock(&vm->invalidated_lock);
2015
2016 return 0;
2017}
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
2035 struct amdgpu_vm *vm,
2036 struct amdgpu_bo *bo)
2037{
2038 struct amdgpu_bo_va *bo_va;
2039
2040 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
2041 if (bo_va == NULL) {
2042 return NULL;
2043 }
2044 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
2045
2046 bo_va->ref_count = 1;
2047 INIT_LIST_HEAD(&bo_va->valids);
2048 INIT_LIST_HEAD(&bo_va->invalids);
2049
2050 if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) &&
2051 (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
2052 bo_va->is_xgmi = true;
2053 mutex_lock(&adev->vm_manager.lock_pstate);
2054
2055 if (++adev->vm_manager.xgmi_map_counter == 1)
2056 amdgpu_xgmi_set_pstate(adev, 1);
2057 mutex_unlock(&adev->vm_manager.lock_pstate);
2058 }
2059
2060 return bo_va;
2061}
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
2074 struct amdgpu_bo_va *bo_va,
2075 struct amdgpu_bo_va_mapping *mapping)
2076{
2077 struct amdgpu_vm *vm = bo_va->base.vm;
2078 struct amdgpu_bo *bo = bo_va->base.bo;
2079
2080 mapping->bo_va = bo_va;
2081 list_add(&mapping->list, &bo_va->invalids);
2082 amdgpu_vm_it_insert(mapping, &vm->va);
2083
2084 if (mapping->flags & AMDGPU_PTE_PRT)
2085 amdgpu_vm_prt_get(adev);
2086
2087 if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
2088 !bo_va->base.moved) {
2089 list_move(&bo_va->base.vm_status, &vm->moved);
2090 }
2091 trace_amdgpu_vm_bo_map(bo_va, mapping);
2092}
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111int amdgpu_vm_bo_map(struct amdgpu_device *adev,
2112 struct amdgpu_bo_va *bo_va,
2113 uint64_t saddr, uint64_t offset,
2114 uint64_t size, uint64_t flags)
2115{
2116 struct amdgpu_bo_va_mapping *mapping, *tmp;
2117 struct amdgpu_bo *bo = bo_va->base.bo;
2118 struct amdgpu_vm *vm = bo_va->base.vm;
2119 uint64_t eaddr;
2120
2121
2122 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2123 size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2124 return -EINVAL;
2125
2126
2127 eaddr = saddr + size - 1;
2128 if (saddr >= eaddr ||
2129 (bo && offset + size > amdgpu_bo_size(bo)))
2130 return -EINVAL;
2131
2132 saddr /= AMDGPU_GPU_PAGE_SIZE;
2133 eaddr /= AMDGPU_GPU_PAGE_SIZE;
2134
2135 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2136 if (tmp) {
2137
2138 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
2139 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
2140 tmp->start, tmp->last + 1);
2141 return -EINVAL;
2142 }
2143
2144 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2145 if (!mapping)
2146 return -ENOMEM;
2147
2148 mapping->start = saddr;
2149 mapping->last = eaddr;
2150 mapping->offset = offset;
2151 mapping->flags = flags;
2152
2153 amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2154
2155 return 0;
2156}
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
2177 struct amdgpu_bo_va *bo_va,
2178 uint64_t saddr, uint64_t offset,
2179 uint64_t size, uint64_t flags)
2180{
2181 struct amdgpu_bo_va_mapping *mapping;
2182 struct amdgpu_bo *bo = bo_va->base.bo;
2183 uint64_t eaddr;
2184 int r;
2185
2186
2187 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2188 size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2189 return -EINVAL;
2190
2191
2192 eaddr = saddr + size - 1;
2193 if (saddr >= eaddr ||
2194 (bo && offset + size > amdgpu_bo_size(bo)))
2195 return -EINVAL;
2196
2197
2198 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2199 if (!mapping)
2200 return -ENOMEM;
2201
2202 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
2203 if (r) {
2204 kfree(mapping);
2205 return r;
2206 }
2207
2208 saddr /= AMDGPU_GPU_PAGE_SIZE;
2209 eaddr /= AMDGPU_GPU_PAGE_SIZE;
2210
2211 mapping->start = saddr;
2212 mapping->last = eaddr;
2213 mapping->offset = offset;
2214 mapping->flags = flags;
2215
2216 amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2217
2218 return 0;
2219}
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2236 struct amdgpu_bo_va *bo_va,
2237 uint64_t saddr)
2238{
2239 struct amdgpu_bo_va_mapping *mapping;
2240 struct amdgpu_vm *vm = bo_va->base.vm;
2241 bool valid = true;
2242
2243 saddr /= AMDGPU_GPU_PAGE_SIZE;
2244
2245 list_for_each_entry(mapping, &bo_va->valids, list) {
2246 if (mapping->start == saddr)
2247 break;
2248 }
2249
2250 if (&mapping->list == &bo_va->valids) {
2251 valid = false;
2252
2253 list_for_each_entry(mapping, &bo_va->invalids, list) {
2254 if (mapping->start == saddr)
2255 break;
2256 }
2257
2258 if (&mapping->list == &bo_va->invalids)
2259 return -ENOENT;
2260 }
2261
2262 list_del(&mapping->list);
2263 amdgpu_vm_it_remove(mapping, &vm->va);
2264 mapping->bo_va = NULL;
2265 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2266
2267 if (valid)
2268 list_add(&mapping->list, &vm->freed);
2269 else
2270 amdgpu_vm_free_mapping(adev, vm, mapping,
2271 bo_va->last_pt_update);
2272
2273 return 0;
2274}
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2290 struct amdgpu_vm *vm,
2291 uint64_t saddr, uint64_t size)
2292{
2293 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2294 LIST_HEAD(removed);
2295 uint64_t eaddr;
2296
2297 eaddr = saddr + size - 1;
2298 saddr /= AMDGPU_GPU_PAGE_SIZE;
2299 eaddr /= AMDGPU_GPU_PAGE_SIZE;
2300
2301
2302 before = kzalloc(sizeof(*before), GFP_KERNEL);
2303 if (!before)
2304 return -ENOMEM;
2305 INIT_LIST_HEAD(&before->list);
2306
2307 after = kzalloc(sizeof(*after), GFP_KERNEL);
2308 if (!after) {
2309 kfree(before);
2310 return -ENOMEM;
2311 }
2312 INIT_LIST_HEAD(&after->list);
2313
2314
2315 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2316 while (tmp) {
2317
2318 if (tmp->start < saddr) {
2319 before->start = tmp->start;
2320 before->last = saddr - 1;
2321 before->offset = tmp->offset;
2322 before->flags = tmp->flags;
2323 before->bo_va = tmp->bo_va;
2324 list_add(&before->list, &tmp->bo_va->invalids);
2325 }
2326
2327
2328 if (tmp->last > eaddr) {
2329 after->start = eaddr + 1;
2330 after->last = tmp->last;
2331 after->offset = tmp->offset;
2332 after->offset += after->start - tmp->start;
2333 after->flags = tmp->flags;
2334 after->bo_va = tmp->bo_va;
2335 list_add(&after->list, &tmp->bo_va->invalids);
2336 }
2337
2338 list_del(&tmp->list);
2339 list_add(&tmp->list, &removed);
2340
2341 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2342 }
2343
2344
2345 list_for_each_entry_safe(tmp, next, &removed, list) {
2346 amdgpu_vm_it_remove(tmp, &vm->va);
2347 list_del(&tmp->list);
2348
2349 if (tmp->start < saddr)
2350 tmp->start = saddr;
2351 if (tmp->last > eaddr)
2352 tmp->last = eaddr;
2353
2354 tmp->bo_va = NULL;
2355 list_add(&tmp->list, &vm->freed);
2356 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2357 }
2358
2359
2360 if (!list_empty(&before->list)) {
2361 amdgpu_vm_it_insert(before, &vm->va);
2362 if (before->flags & AMDGPU_PTE_PRT)
2363 amdgpu_vm_prt_get(adev);
2364 } else {
2365 kfree(before);
2366 }
2367
2368
2369 if (!list_empty(&after->list)) {
2370 amdgpu_vm_it_insert(after, &vm->va);
2371 if (after->flags & AMDGPU_PTE_PRT)
2372 amdgpu_vm_prt_get(adev);
2373 } else {
2374 kfree(after);
2375 }
2376
2377 return 0;
2378}
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2393 uint64_t addr)
2394{
2395 return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2396}
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2407{
2408 struct amdgpu_bo_va_mapping *mapping;
2409
2410 if (!trace_amdgpu_vm_bo_cs_enabled())
2411 return;
2412
2413 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
2414 mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
2415 if (mapping->bo_va && mapping->bo_va->base.bo) {
2416 struct amdgpu_bo *bo;
2417
2418 bo = mapping->bo_va->base.bo;
2419 if (READ_ONCE(bo->tbo.resv->lock.ctx) != ticket)
2420 continue;
2421 }
2422
2423 trace_amdgpu_vm_bo_cs(mapping);
2424 }
2425}
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2438 struct amdgpu_bo_va *bo_va)
2439{
2440 struct amdgpu_bo_va_mapping *mapping, *next;
2441 struct amdgpu_bo *bo = bo_va->base.bo;
2442 struct amdgpu_vm *vm = bo_va->base.vm;
2443 struct amdgpu_vm_bo_base **base;
2444
2445 if (bo) {
2446 if (bo->tbo.resv == vm->root.base.bo->tbo.resv)
2447 vm->bulk_moveable = false;
2448
2449 for (base = &bo_va->base.bo->vm_bo; *base;
2450 base = &(*base)->next) {
2451 if (*base != &bo_va->base)
2452 continue;
2453
2454 *base = bo_va->base.next;
2455 break;
2456 }
2457 }
2458
2459 spin_lock(&vm->invalidated_lock);
2460 list_del(&bo_va->base.vm_status);
2461 spin_unlock(&vm->invalidated_lock);
2462
2463 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2464 list_del(&mapping->list);
2465 amdgpu_vm_it_remove(mapping, &vm->va);
2466 mapping->bo_va = NULL;
2467 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2468 list_add(&mapping->list, &vm->freed);
2469 }
2470 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2471 list_del(&mapping->list);
2472 amdgpu_vm_it_remove(mapping, &vm->va);
2473 amdgpu_vm_free_mapping(adev, vm, mapping,
2474 bo_va->last_pt_update);
2475 }
2476
2477 dma_fence_put(bo_va->last_pt_update);
2478
2479 if (bo && bo_va->is_xgmi) {
2480 mutex_lock(&adev->vm_manager.lock_pstate);
2481 if (--adev->vm_manager.xgmi_map_counter == 0)
2482 amdgpu_xgmi_set_pstate(adev, 0);
2483 mutex_unlock(&adev->vm_manager.lock_pstate);
2484 }
2485
2486 kfree(bo_va);
2487}
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2499 struct amdgpu_bo *bo, bool evicted)
2500{
2501 struct amdgpu_vm_bo_base *bo_base;
2502
2503
2504 if (bo->parent && bo->parent->shadow == bo)
2505 bo = bo->parent;
2506
2507 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2508 struct amdgpu_vm *vm = bo_base->vm;
2509
2510 if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
2511 amdgpu_vm_bo_evicted(bo_base);
2512 continue;
2513 }
2514
2515 if (bo_base->moved)
2516 continue;
2517 bo_base->moved = true;
2518
2519 if (bo->tbo.type == ttm_bo_type_kernel)
2520 amdgpu_vm_bo_relocated(bo_base);
2521 else if (bo->tbo.resv == vm->root.base.bo->tbo.resv)
2522 amdgpu_vm_bo_moved(bo_base);
2523 else
2524 amdgpu_vm_bo_invalidated(bo_base);
2525 }
2526}
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2537{
2538
2539 unsigned bits = ilog2(vm_size) + 18;
2540
2541
2542
2543 if (vm_size <= 8)
2544 return (bits - 9);
2545 else
2546 return ((bits + 3) / 2);
2547}
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2560 uint32_t fragment_size_default, unsigned max_level,
2561 unsigned max_bits)
2562{
2563 unsigned int max_size = 1 << (max_bits - 30);
2564 unsigned int vm_size;
2565 uint64_t tmp;
2566
2567
2568 if (amdgpu_vm_size != -1) {
2569 vm_size = amdgpu_vm_size;
2570 if (vm_size > max_size) {
2571 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2572 amdgpu_vm_size, max_size);
2573 vm_size = max_size;
2574 }
2575 } else {
2576 struct sysinfo si;
2577 unsigned int phys_ram_gb;
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594 si_meminfo(&si);
2595 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2596 (1 << 30) - 1) >> 30;
2597 vm_size = roundup_pow_of_two(
2598 min(max(phys_ram_gb * 3, min_vm_size), max_size));
2599 }
2600
2601 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2602
2603 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2604 if (amdgpu_vm_block_size != -1)
2605 tmp >>= amdgpu_vm_block_size - 9;
2606 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2607 adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2608 switch (adev->vm_manager.num_level) {
2609 case 3:
2610 adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2611 break;
2612 case 2:
2613 adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2614 break;
2615 case 1:
2616 adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2617 break;
2618 default:
2619 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2620 }
2621
2622 if (amdgpu_vm_block_size != -1)
2623 adev->vm_manager.block_size =
2624 min((unsigned)amdgpu_vm_block_size, max_bits
2625 - AMDGPU_GPU_PAGE_SHIFT
2626 - 9 * adev->vm_manager.num_level);
2627 else if (adev->vm_manager.num_level > 1)
2628 adev->vm_manager.block_size = 9;
2629 else
2630 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2631
2632 if (amdgpu_vm_fragment_size == -1)
2633 adev->vm_manager.fragment_size = fragment_size_default;
2634 else
2635 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2636
2637 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2638 vm_size, adev->vm_manager.num_level + 1,
2639 adev->vm_manager.block_size,
2640 adev->vm_manager.fragment_size);
2641}
2642
2643
2644
2645
2646
2647
2648
2649long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2650{
2651 return reservation_object_wait_timeout_rcu(vm->root.base.bo->tbo.resv,
2652 true, true, timeout);
2653}
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2669 int vm_context, unsigned int pasid)
2670{
2671 struct amdgpu_bo_param bp;
2672 struct amdgpu_bo *root;
2673 int r, i;
2674
2675 vm->va = RB_ROOT_CACHED;
2676 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2677 vm->reserved_vmid[i] = NULL;
2678 INIT_LIST_HEAD(&vm->evicted);
2679 INIT_LIST_HEAD(&vm->relocated);
2680 INIT_LIST_HEAD(&vm->moved);
2681 INIT_LIST_HEAD(&vm->idle);
2682 INIT_LIST_HEAD(&vm->invalidated);
2683 spin_lock_init(&vm->invalidated_lock);
2684 INIT_LIST_HEAD(&vm->freed);
2685
2686
2687 r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs,
2688 adev->vm_manager.vm_pte_num_rqs, NULL);
2689 if (r)
2690 return r;
2691
2692 vm->pte_support_ats = false;
2693
2694 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2695 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2696 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2697
2698 if (adev->asic_type == CHIP_RAVEN)
2699 vm->pte_support_ats = true;
2700 } else {
2701 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2702 AMDGPU_VM_USE_CPU_FOR_GFX);
2703 }
2704 DRM_DEBUG_DRIVER("VM update mode is %s\n",
2705 vm->use_cpu_for_update ? "CPU" : "SDMA");
2706 WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2707 "CPU update of VM recommended only for large BAR system\n");
2708
2709 if (vm->use_cpu_for_update)
2710 vm->update_funcs = &amdgpu_vm_cpu_funcs;
2711 else
2712 vm->update_funcs = &amdgpu_vm_sdma_funcs;
2713 vm->last_update = NULL;
2714
2715 amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp);
2716 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
2717 bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW;
2718 r = amdgpu_bo_create(adev, &bp, &root);
2719 if (r)
2720 goto error_free_sched_entity;
2721
2722 r = amdgpu_bo_reserve(root, true);
2723 if (r)
2724 goto error_free_root;
2725
2726 r = reservation_object_reserve_shared(root->tbo.resv, 1);
2727 if (r)
2728 goto error_unreserve;
2729
2730 amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
2731
2732 r = amdgpu_vm_clear_bo(adev, vm, root);
2733 if (r)
2734 goto error_unreserve;
2735
2736 amdgpu_bo_unreserve(vm->root.base.bo);
2737
2738 if (pasid) {
2739 unsigned long flags;
2740
2741 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2742 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2743 GFP_ATOMIC);
2744 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2745 if (r < 0)
2746 goto error_free_root;
2747
2748 vm->pasid = pasid;
2749 }
2750
2751 INIT_KFIFO(vm->faults);
2752
2753 return 0;
2754
2755error_unreserve:
2756 amdgpu_bo_unreserve(vm->root.base.bo);
2757
2758error_free_root:
2759 amdgpu_bo_unref(&vm->root.base.bo->shadow);
2760 amdgpu_bo_unref(&vm->root.base.bo);
2761 vm->root.base.bo = NULL;
2762
2763error_free_sched_entity:
2764 drm_sched_entity_destroy(&vm->entity);
2765
2766 return r;
2767}
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev,
2783 struct amdgpu_vm *vm)
2784{
2785 enum amdgpu_vm_level root = adev->vm_manager.root_level;
2786 unsigned int entries = amdgpu_vm_num_entries(adev, root);
2787 unsigned int i = 0;
2788
2789 if (!(vm->root.entries))
2790 return 0;
2791
2792 for (i = 0; i < entries; i++) {
2793 if (vm->root.entries[i].base.bo)
2794 return -EINVAL;
2795 }
2796
2797 return 0;
2798}
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid)
2821{
2822 bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2823 int r;
2824
2825 r = amdgpu_bo_reserve(vm->root.base.bo, true);
2826 if (r)
2827 return r;
2828
2829
2830 r = amdgpu_vm_check_clean_reserved(adev, vm);
2831 if (r)
2832 goto unreserve_bo;
2833
2834 if (pasid) {
2835 unsigned long flags;
2836
2837 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2838 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2839 GFP_ATOMIC);
2840 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2841
2842 if (r == -ENOSPC)
2843 goto unreserve_bo;
2844 r = 0;
2845 }
2846
2847
2848
2849
2850 if (pte_support_ats != vm->pte_support_ats) {
2851 vm->pte_support_ats = pte_support_ats;
2852 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo);
2853 if (r)
2854 goto free_idr;
2855 }
2856
2857
2858 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2859 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2860 DRM_DEBUG_DRIVER("VM update mode is %s\n",
2861 vm->use_cpu_for_update ? "CPU" : "SDMA");
2862 WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2863 "CPU update of VM recommended only for large BAR system\n");
2864
2865 if (vm->pasid) {
2866 unsigned long flags;
2867
2868 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2869 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2870 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2871
2872
2873
2874
2875 amdgpu_pasid_free(vm->pasid);
2876 vm->pasid = 0;
2877 }
2878
2879
2880 amdgpu_bo_unref(&vm->root.base.bo->shadow);
2881
2882 if (pasid)
2883 vm->pasid = pasid;
2884
2885 goto unreserve_bo;
2886
2887free_idr:
2888 if (pasid) {
2889 unsigned long flags;
2890
2891 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2892 idr_remove(&adev->vm_manager.pasid_idr, pasid);
2893 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2894 }
2895unreserve_bo:
2896 amdgpu_bo_unreserve(vm->root.base.bo);
2897 return r;
2898}
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2909{
2910 if (vm->pasid) {
2911 unsigned long flags;
2912
2913 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2914 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2915 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2916 }
2917 vm->pasid = 0;
2918}
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2930{
2931 struct amdgpu_bo_va_mapping *mapping, *tmp;
2932 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2933 struct amdgpu_bo *root;
2934 int i, r;
2935
2936 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2937
2938 if (vm->pasid) {
2939 unsigned long flags;
2940
2941 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2942 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2943 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2944 }
2945
2946 drm_sched_entity_destroy(&vm->entity);
2947
2948 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2949 dev_err(adev->dev, "still active bo inside vm\n");
2950 }
2951 rbtree_postorder_for_each_entry_safe(mapping, tmp,
2952 &vm->va.rb_root, rb) {
2953
2954
2955
2956 list_del(&mapping->list);
2957 kfree(mapping);
2958 }
2959 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2960 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2961 amdgpu_vm_prt_fini(adev, vm);
2962 prt_fini_needed = false;
2963 }
2964
2965 list_del(&mapping->list);
2966 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2967 }
2968
2969 root = amdgpu_bo_ref(vm->root.base.bo);
2970 r = amdgpu_bo_reserve(root, true);
2971 if (r) {
2972 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
2973 } else {
2974 amdgpu_vm_free_pts(adev, vm, NULL);
2975 amdgpu_bo_unreserve(root);
2976 }
2977 amdgpu_bo_unref(&root);
2978 WARN_ON(vm->root.base.bo);
2979 dma_fence_put(vm->last_update);
2980 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2981 amdgpu_vmid_free_reserved(adev, vm, i);
2982}
2983
2984
2985
2986
2987
2988
2989
2990
2991void amdgpu_vm_manager_init(struct amdgpu_device *adev)
2992{
2993 unsigned i;
2994
2995 amdgpu_vmid_mgr_init(adev);
2996
2997 adev->vm_manager.fence_context =
2998 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2999 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
3000 adev->vm_manager.seqno[i] = 0;
3001
3002 spin_lock_init(&adev->vm_manager.prt_lock);
3003 atomic_set(&adev->vm_manager.num_prt_users, 0);
3004
3005
3006
3007
3008#ifdef CONFIG_X86_64
3009 if (amdgpu_vm_update_mode == -1) {
3010 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
3011 adev->vm_manager.vm_update_mode =
3012 AMDGPU_VM_USE_CPU_FOR_COMPUTE;
3013 else
3014 adev->vm_manager.vm_update_mode = 0;
3015 } else
3016 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
3017#else
3018 adev->vm_manager.vm_update_mode = 0;
3019#endif
3020
3021 idr_init(&adev->vm_manager.pasid_idr);
3022 spin_lock_init(&adev->vm_manager.pasid_lock);
3023
3024 adev->vm_manager.xgmi_map_counter = 0;
3025 mutex_init(&adev->vm_manager.lock_pstate);
3026}
3027
3028
3029
3030
3031
3032
3033
3034
3035void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
3036{
3037 WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
3038 idr_destroy(&adev->vm_manager.pasid_idr);
3039
3040 amdgpu_vmid_mgr_fini(adev);
3041}
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
3054{
3055 union drm_amdgpu_vm *args = data;
3056 struct amdgpu_device *adev = dev->dev_private;
3057 struct amdgpu_fpriv *fpriv = filp->driver_priv;
3058 int r;
3059
3060 switch (args->in.op) {
3061 case AMDGPU_VM_OP_RESERVE_VMID:
3062
3063 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
3064 if (r)
3065 return r;
3066 break;
3067 case AMDGPU_VM_OP_UNRESERVE_VMID:
3068 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
3069 break;
3070 default:
3071 return -EINVAL;
3072 }
3073
3074 return 0;
3075}
3076
3077
3078
3079
3080
3081
3082
3083
3084void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
3085 struct amdgpu_task_info *task_info)
3086{
3087 struct amdgpu_vm *vm;
3088 unsigned long flags;
3089
3090 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
3091
3092 vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
3093 if (vm)
3094 *task_info = vm->task_info;
3095
3096 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
3097}
3098
3099
3100
3101
3102
3103
3104void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
3105{
3106 if (!vm->task_info.pid) {
3107 vm->task_info.pid = current->pid;
3108 get_task_comm(vm->task_info.task_name, current);
3109
3110 if (current->group_leader->mm == current->mm) {
3111 vm->task_info.tgid = current->group_leader->pid;
3112 get_task_comm(vm->task_info.process_name, current->group_leader);
3113 }
3114 }
3115}
3116