1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#include <linux/list.h>
33#include <linux/slab.h>
34#include <drm/drmP.h>
35#include <drm/amdgpu_drm.h>
36#include <drm/drm_cache.h>
37#include "amdgpu.h"
38#include "amdgpu_trace.h"
39
40static bool amdgpu_need_backup(struct amdgpu_device *adev)
41{
42 if (adev->flags & AMD_IS_APU)
43 return false;
44
45 if (amdgpu_gpu_recovery == 0 ||
46 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev)))
47 return false;
48
49 return true;
50}
51
52static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
53{
54 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
55 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
56
57 amdgpu_bo_kunmap(bo);
58
59 if (bo->gem_base.import_attach)
60 drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
61 drm_gem_object_release(&bo->gem_base);
62 amdgpu_bo_unref(&bo->parent);
63 if (!list_empty(&bo->shadow_list)) {
64 mutex_lock(&adev->shadow_list_lock);
65 list_del_init(&bo->shadow_list);
66 mutex_unlock(&adev->shadow_list_lock);
67 }
68 kfree(bo->metadata);
69 kfree(bo);
70}
71
72bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
73{
74 if (bo->destroy == &amdgpu_ttm_bo_destroy)
75 return true;
76 return false;
77}
78
79void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
80{
81 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
82 struct ttm_placement *placement = &abo->placement;
83 struct ttm_place *places = abo->placements;
84 u64 flags = abo->flags;
85 u32 c = 0;
86
87 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
88 unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
89
90 places[c].fpfn = 0;
91 places[c].lpfn = 0;
92 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
93 TTM_PL_FLAG_VRAM;
94
95 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
96 places[c].lpfn = visible_pfn;
97 else
98 places[c].flags |= TTM_PL_FLAG_TOPDOWN;
99
100 if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
101 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
102 c++;
103 }
104
105 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
106 places[c].fpfn = 0;
107 if (flags & AMDGPU_GEM_CREATE_SHADOW)
108 places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT;
109 else
110 places[c].lpfn = 0;
111 places[c].flags = TTM_PL_FLAG_TT;
112 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
113 places[c].flags |= TTM_PL_FLAG_WC |
114 TTM_PL_FLAG_UNCACHED;
115 else
116 places[c].flags |= TTM_PL_FLAG_CACHED;
117 c++;
118 }
119
120 if (domain & AMDGPU_GEM_DOMAIN_CPU) {
121 places[c].fpfn = 0;
122 places[c].lpfn = 0;
123 places[c].flags = TTM_PL_FLAG_SYSTEM;
124 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
125 places[c].flags |= TTM_PL_FLAG_WC |
126 TTM_PL_FLAG_UNCACHED;
127 else
128 places[c].flags |= TTM_PL_FLAG_CACHED;
129 c++;
130 }
131
132 if (domain & AMDGPU_GEM_DOMAIN_GDS) {
133 places[c].fpfn = 0;
134 places[c].lpfn = 0;
135 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
136 c++;
137 }
138
139 if (domain & AMDGPU_GEM_DOMAIN_GWS) {
140 places[c].fpfn = 0;
141 places[c].lpfn = 0;
142 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
143 c++;
144 }
145
146 if (domain & AMDGPU_GEM_DOMAIN_OA) {
147 places[c].fpfn = 0;
148 places[c].lpfn = 0;
149 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
150 c++;
151 }
152
153 if (!c) {
154 places[c].fpfn = 0;
155 places[c].lpfn = 0;
156 places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
157 c++;
158 }
159
160 placement->num_placement = c;
161 placement->placement = places;
162
163 placement->num_busy_placement = c;
164 placement->busy_placement = places;
165}
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
184 unsigned long size, int align,
185 u32 domain, struct amdgpu_bo **bo_ptr,
186 u64 *gpu_addr, void **cpu_addr)
187{
188 bool free = false;
189 int r;
190
191 if (!*bo_ptr) {
192 r = amdgpu_bo_create(adev, size, align, true, domain,
193 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
194 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
195 NULL, NULL, 0, bo_ptr);
196 if (r) {
197 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
198 r);
199 return r;
200 }
201 free = true;
202 }
203
204 r = amdgpu_bo_reserve(*bo_ptr, false);
205 if (r) {
206 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
207 goto error_free;
208 }
209
210 r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
211 if (r) {
212 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
213 goto error_unreserve;
214 }
215
216 if (cpu_addr) {
217 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
218 if (r) {
219 dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
220 goto error_unreserve;
221 }
222 }
223
224 return 0;
225
226error_unreserve:
227 amdgpu_bo_unreserve(*bo_ptr);
228
229error_free:
230 if (free)
231 amdgpu_bo_unref(bo_ptr);
232
233 return r;
234}
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
252 unsigned long size, int align,
253 u32 domain, struct amdgpu_bo **bo_ptr,
254 u64 *gpu_addr, void **cpu_addr)
255{
256 int r;
257
258 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
259 gpu_addr, cpu_addr);
260
261 if (r)
262 return r;
263
264 amdgpu_bo_unreserve(*bo_ptr);
265
266 return 0;
267}
268
269
270
271
272
273
274
275
276void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
277 void **cpu_addr)
278{
279 if (*bo == NULL)
280 return;
281
282 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
283 if (cpu_addr)
284 amdgpu_bo_kunmap(*bo);
285
286 amdgpu_bo_unpin(*bo);
287 amdgpu_bo_unreserve(*bo);
288 }
289 amdgpu_bo_unref(bo);
290
291 if (gpu_addr)
292 *gpu_addr = 0;
293
294 if (cpu_addr)
295 *cpu_addr = NULL;
296}
297
298
299static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
300 unsigned long size, u32 domain)
301{
302 struct ttm_mem_type_manager *man = NULL;
303
304
305
306
307
308 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
309 man = &adev->mman.bdev.man[TTM_PL_TT];
310
311 if (size < (man->size << PAGE_SHIFT))
312 return true;
313 else
314 goto fail;
315 }
316
317 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
318 man = &adev->mman.bdev.man[TTM_PL_VRAM];
319
320 if (size < (man->size << PAGE_SHIFT))
321 return true;
322 else
323 goto fail;
324 }
325
326
327
328 return true;
329
330fail:
331 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
332 man->size << PAGE_SHIFT);
333 return false;
334}
335
336static int amdgpu_bo_do_create(struct amdgpu_device *adev,
337 unsigned long size, int byte_align,
338 bool kernel, u32 domain, u64 flags,
339 struct sg_table *sg,
340 struct reservation_object *resv,
341 uint64_t init_value,
342 struct amdgpu_bo **bo_ptr)
343{
344 struct ttm_operation_ctx ctx = {
345 .interruptible = !kernel,
346 .no_wait_gpu = false,
347 .allow_reserved_eviction = true,
348 .resv = resv
349 };
350 struct amdgpu_bo *bo;
351 enum ttm_bo_type type;
352 unsigned long page_align;
353 size_t acc_size;
354 int r;
355
356 page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
357 size = ALIGN(size, PAGE_SIZE);
358
359 if (!amdgpu_bo_validate_size(adev, size, domain))
360 return -ENOMEM;
361
362 if (kernel) {
363 type = ttm_bo_type_kernel;
364 } else if (sg) {
365 type = ttm_bo_type_sg;
366 } else {
367 type = ttm_bo_type_device;
368 }
369 *bo_ptr = NULL;
370
371 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
372 sizeof(struct amdgpu_bo));
373
374 bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
375 if (bo == NULL)
376 return -ENOMEM;
377 r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
378 if (unlikely(r)) {
379 kfree(bo);
380 return r;
381 }
382 INIT_LIST_HEAD(&bo->shadow_list);
383 INIT_LIST_HEAD(&bo->va);
384 bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
385 AMDGPU_GEM_DOMAIN_GTT |
386 AMDGPU_GEM_DOMAIN_CPU |
387 AMDGPU_GEM_DOMAIN_GDS |
388 AMDGPU_GEM_DOMAIN_GWS |
389 AMDGPU_GEM_DOMAIN_OA);
390 bo->allowed_domains = bo->preferred_domains;
391 if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
392 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
393
394 bo->flags = flags;
395
396#ifdef CONFIG_X86_32
397
398
399
400 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
401#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
402
403
404
405
406
407#ifndef CONFIG_COMPILE_TEST
408#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
409 thanks to write-combining
410#endif
411
412 if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
413 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
414 "better performance thanks to write-combining\n");
415 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
416#else
417
418
419
420 if (!drm_arch_can_wc_memory())
421 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
422#endif
423
424 bo->tbo.bdev = &adev->mman.bdev;
425 amdgpu_ttm_placement_from_domain(bo, domain);
426
427 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
428 &bo->placement, page_align, &ctx, NULL,
429 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
430 if (unlikely(r != 0))
431 return r;
432
433 if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
434 bo->tbo.mem.mem_type == TTM_PL_VRAM &&
435 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
436 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
437 ctx.bytes_moved);
438 else
439 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
440
441 if (kernel)
442 bo->tbo.priority = 1;
443
444 if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
445 bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
446 struct dma_fence *fence;
447
448 r = amdgpu_fill_buffer(bo, init_value, bo->tbo.resv, &fence);
449 if (unlikely(r))
450 goto fail_unreserve;
451
452 amdgpu_bo_fence(bo, fence, false);
453 dma_fence_put(bo->tbo.moving);
454 bo->tbo.moving = dma_fence_get(fence);
455 dma_fence_put(fence);
456 }
457 if (!resv)
458 amdgpu_bo_unreserve(bo);
459 *bo_ptr = bo;
460
461 trace_amdgpu_bo_create(bo);
462
463
464 if (type == ttm_bo_type_device)
465 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
466
467 return 0;
468
469fail_unreserve:
470 if (!resv)
471 ww_mutex_unlock(&bo->tbo.resv->lock);
472 amdgpu_bo_unref(&bo);
473 return r;
474}
475
476static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
477 unsigned long size, int byte_align,
478 struct amdgpu_bo *bo)
479{
480 int r;
481
482 if (bo->shadow)
483 return 0;
484
485 r = amdgpu_bo_do_create(adev, size, byte_align, true,
486 AMDGPU_GEM_DOMAIN_GTT,
487 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
488 AMDGPU_GEM_CREATE_SHADOW,
489 NULL, bo->tbo.resv, 0,
490 &bo->shadow);
491 if (!r) {
492 bo->shadow->parent = amdgpu_bo_ref(bo);
493 mutex_lock(&adev->shadow_list_lock);
494 list_add_tail(&bo->shadow_list, &adev->shadow_list);
495 mutex_unlock(&adev->shadow_list_lock);
496 }
497
498 return r;
499}
500
501
502
503
504int amdgpu_bo_create(struct amdgpu_device *adev,
505 unsigned long size, int byte_align,
506 bool kernel, u32 domain, u64 flags,
507 struct sg_table *sg,
508 struct reservation_object *resv,
509 uint64_t init_value,
510 struct amdgpu_bo **bo_ptr)
511{
512 uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
513 int r;
514
515 r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,
516 parent_flags, sg, resv, init_value, bo_ptr);
517 if (r)
518 return r;
519
520 if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {
521 if (!resv)
522 WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
523 NULL));
524
525 r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
526
527 if (!resv)
528 reservation_object_unlock((*bo_ptr)->tbo.resv);
529
530 if (r)
531 amdgpu_bo_unref(bo_ptr);
532 }
533
534 return r;
535}
536
537int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
538 struct amdgpu_ring *ring,
539 struct amdgpu_bo *bo,
540 struct reservation_object *resv,
541 struct dma_fence **fence,
542 bool direct)
543
544{
545 struct amdgpu_bo *shadow = bo->shadow;
546 uint64_t bo_addr, shadow_addr;
547 int r;
548
549 if (!shadow)
550 return -EINVAL;
551
552 bo_addr = amdgpu_bo_gpu_offset(bo);
553 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
554
555 r = reservation_object_reserve_shared(bo->tbo.resv);
556 if (r)
557 goto err;
558
559 r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
560 amdgpu_bo_size(bo), resv, fence,
561 direct, false);
562 if (!r)
563 amdgpu_bo_fence(bo, *fence, true);
564
565err:
566 return r;
567}
568
569int amdgpu_bo_validate(struct amdgpu_bo *bo)
570{
571 struct ttm_operation_ctx ctx = { false, false };
572 uint32_t domain;
573 int r;
574
575 if (bo->pin_count)
576 return 0;
577
578 domain = bo->preferred_domains;
579
580retry:
581 amdgpu_ttm_placement_from_domain(bo, domain);
582 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
583 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
584 domain = bo->allowed_domains;
585 goto retry;
586 }
587
588 return r;
589}
590
591int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
592 struct amdgpu_ring *ring,
593 struct amdgpu_bo *bo,
594 struct reservation_object *resv,
595 struct dma_fence **fence,
596 bool direct)
597
598{
599 struct amdgpu_bo *shadow = bo->shadow;
600 uint64_t bo_addr, shadow_addr;
601 int r;
602
603 if (!shadow)
604 return -EINVAL;
605
606 bo_addr = amdgpu_bo_gpu_offset(bo);
607 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
608
609 r = reservation_object_reserve_shared(bo->tbo.resv);
610 if (r)
611 goto err;
612
613 r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
614 amdgpu_bo_size(bo), resv, fence,
615 direct, false);
616 if (!r)
617 amdgpu_bo_fence(bo, *fence, true);
618
619err:
620 return r;
621}
622
623int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
624{
625 void *kptr;
626 long r;
627
628 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
629 return -EPERM;
630
631 kptr = amdgpu_bo_kptr(bo);
632 if (kptr) {
633 if (ptr)
634 *ptr = kptr;
635 return 0;
636 }
637
638 r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
639 MAX_SCHEDULE_TIMEOUT);
640 if (r < 0)
641 return r;
642
643 r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
644 if (r)
645 return r;
646
647 if (ptr)
648 *ptr = amdgpu_bo_kptr(bo);
649
650 return 0;
651}
652
653void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
654{
655 bool is_iomem;
656
657 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
658}
659
660void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
661{
662 if (bo->kmap.bo)
663 ttm_bo_kunmap(&bo->kmap);
664}
665
666struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
667{
668 if (bo == NULL)
669 return NULL;
670
671 ttm_bo_reference(&bo->tbo);
672 return bo;
673}
674
675void amdgpu_bo_unref(struct amdgpu_bo **bo)
676{
677 struct ttm_buffer_object *tbo;
678
679 if ((*bo) == NULL)
680 return;
681
682 tbo = &((*bo)->tbo);
683 ttm_bo_unref(&tbo);
684 if (tbo == NULL)
685 *bo = NULL;
686}
687
688int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
689 u64 min_offset, u64 max_offset,
690 u64 *gpu_addr)
691{
692 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
693 struct ttm_operation_ctx ctx = { false, false };
694 int r, i;
695
696 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
697 return -EPERM;
698
699 if (WARN_ON_ONCE(min_offset > max_offset))
700 return -EINVAL;
701
702
703 if (bo->prime_shared_count && (domain == AMDGPU_GEM_DOMAIN_VRAM))
704 return -EINVAL;
705
706 if (bo->pin_count) {
707 uint32_t mem_type = bo->tbo.mem.mem_type;
708
709 if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
710 return -EINVAL;
711
712 bo->pin_count++;
713 if (gpu_addr)
714 *gpu_addr = amdgpu_bo_gpu_offset(bo);
715
716 if (max_offset != 0) {
717 u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
718 WARN_ON_ONCE(max_offset <
719 (amdgpu_bo_gpu_offset(bo) - domain_start));
720 }
721
722 return 0;
723 }
724
725 bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
726
727 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
728 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
729 amdgpu_ttm_placement_from_domain(bo, domain);
730 for (i = 0; i < bo->placement.num_placement; i++) {
731 unsigned fpfn, lpfn;
732
733 fpfn = min_offset >> PAGE_SHIFT;
734 lpfn = max_offset >> PAGE_SHIFT;
735
736 if (fpfn > bo->placements[i].fpfn)
737 bo->placements[i].fpfn = fpfn;
738 if (!bo->placements[i].lpfn ||
739 (lpfn && lpfn < bo->placements[i].lpfn))
740 bo->placements[i].lpfn = lpfn;
741 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
742 }
743
744 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
745 if (unlikely(r)) {
746 dev_err(adev->dev, "%p pin failed\n", bo);
747 goto error;
748 }
749
750 r = amdgpu_ttm_alloc_gart(&bo->tbo);
751 if (unlikely(r)) {
752 dev_err(adev->dev, "%p bind failed\n", bo);
753 goto error;
754 }
755
756 bo->pin_count = 1;
757 if (gpu_addr != NULL)
758 *gpu_addr = amdgpu_bo_gpu_offset(bo);
759
760 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
761 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
762 adev->vram_pin_size += amdgpu_bo_size(bo);
763 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
764 adev->invisible_pin_size += amdgpu_bo_size(bo);
765 } else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
766 adev->gart_pin_size += amdgpu_bo_size(bo);
767 }
768
769error:
770 return r;
771}
772
773int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
774{
775 return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
776}
777
778int amdgpu_bo_unpin(struct amdgpu_bo *bo)
779{
780 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
781 struct ttm_operation_ctx ctx = { false, false };
782 int r, i;
783
784 if (!bo->pin_count) {
785 dev_warn(adev->dev, "%p unpin not necessary\n", bo);
786 return 0;
787 }
788 bo->pin_count--;
789 if (bo->pin_count)
790 return 0;
791 for (i = 0; i < bo->placement.num_placement; i++) {
792 bo->placements[i].lpfn = 0;
793 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
794 }
795 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
796 if (unlikely(r)) {
797 dev_err(adev->dev, "%p validate failed for unpin\n", bo);
798 goto error;
799 }
800
801 if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
802 adev->vram_pin_size -= amdgpu_bo_size(bo);
803 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
804 adev->invisible_pin_size -= amdgpu_bo_size(bo);
805 } else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
806 adev->gart_pin_size -= amdgpu_bo_size(bo);
807 }
808
809error:
810 return r;
811}
812
813int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
814{
815
816 if (0 && (adev->flags & AMD_IS_APU)) {
817
818 return 0;
819 }
820 return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
821}
822
823static const char *amdgpu_vram_names[] = {
824 "UNKNOWN",
825 "GDDR1",
826 "DDR2",
827 "GDDR3",
828 "GDDR4",
829 "GDDR5",
830 "HBM",
831 "DDR3"
832};
833
834int amdgpu_bo_init(struct amdgpu_device *adev)
835{
836
837 arch_io_reserve_memtype_wc(adev->mc.aper_base,
838 adev->mc.aper_size);
839
840
841 adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
842 adev->mc.aper_size);
843 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
844 adev->mc.mc_vram_size >> 20,
845 (unsigned long long)adev->mc.aper_size >> 20);
846 DRM_INFO("RAM width %dbits %s\n",
847 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
848 return amdgpu_ttm_init(adev);
849}
850
851void amdgpu_bo_fini(struct amdgpu_device *adev)
852{
853 amdgpu_ttm_fini(adev);
854 arch_phys_wc_del(adev->mc.vram_mtrr);
855 arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size);
856}
857
858int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
859 struct vm_area_struct *vma)
860{
861 return ttm_fbdev_mmap(vma, &bo->tbo);
862}
863
864int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
865{
866 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
867
868 if (adev->family <= AMDGPU_FAMILY_CZ &&
869 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
870 return -EINVAL;
871
872 bo->tiling_flags = tiling_flags;
873 return 0;
874}
875
876void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
877{
878 lockdep_assert_held(&bo->tbo.resv->lock.base);
879
880 if (tiling_flags)
881 *tiling_flags = bo->tiling_flags;
882}
883
884int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
885 uint32_t metadata_size, uint64_t flags)
886{
887 void *buffer;
888
889 if (!metadata_size) {
890 if (bo->metadata_size) {
891 kfree(bo->metadata);
892 bo->metadata = NULL;
893 bo->metadata_size = 0;
894 }
895 return 0;
896 }
897
898 if (metadata == NULL)
899 return -EINVAL;
900
901 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
902 if (buffer == NULL)
903 return -ENOMEM;
904
905 kfree(bo->metadata);
906 bo->metadata_flags = flags;
907 bo->metadata = buffer;
908 bo->metadata_size = metadata_size;
909
910 return 0;
911}
912
913int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
914 size_t buffer_size, uint32_t *metadata_size,
915 uint64_t *flags)
916{
917 if (!buffer && !metadata_size)
918 return -EINVAL;
919
920 if (buffer) {
921 if (buffer_size < bo->metadata_size)
922 return -EINVAL;
923
924 if (bo->metadata_size)
925 memcpy(buffer, bo->metadata, bo->metadata_size);
926 }
927
928 if (metadata_size)
929 *metadata_size = bo->metadata_size;
930 if (flags)
931 *flags = bo->metadata_flags;
932
933 return 0;
934}
935
936void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
937 bool evict,
938 struct ttm_mem_reg *new_mem)
939{
940 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
941 struct amdgpu_bo *abo;
942 struct ttm_mem_reg *old_mem = &bo->mem;
943
944 if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
945 return;
946
947 abo = ttm_to_amdgpu_bo(bo);
948 amdgpu_vm_bo_invalidate(adev, abo, evict);
949
950 amdgpu_bo_kunmap(abo);
951
952
953 if (evict)
954 atomic64_inc(&adev->num_evictions);
955
956
957 if (!new_mem)
958 return;
959
960
961 trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
962}
963
964int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
965{
966 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
967 struct ttm_operation_ctx ctx = { false, false };
968 struct amdgpu_bo *abo;
969 unsigned long offset, size;
970 int r;
971
972 if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
973 return 0;
974
975 abo = ttm_to_amdgpu_bo(bo);
976
977
978 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
979
980 if (bo->mem.mem_type != TTM_PL_VRAM)
981 return 0;
982
983 size = bo->mem.num_pages << PAGE_SHIFT;
984 offset = bo->mem.start << PAGE_SHIFT;
985 if ((offset + size) <= adev->mc.visible_vram_size)
986 return 0;
987
988
989 if (abo->pin_count > 0)
990 return -EINVAL;
991
992
993 atomic64_inc(&adev->num_vram_cpu_page_faults);
994 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
995 AMDGPU_GEM_DOMAIN_GTT);
996
997
998 abo->placement.num_busy_placement = 1;
999 abo->placement.busy_placement = &abo->placements[1];
1000
1001 r = ttm_bo_validate(bo, &abo->placement, &ctx);
1002 if (unlikely(r != 0))
1003 return r;
1004
1005 offset = bo->mem.start << PAGE_SHIFT;
1006
1007 if (bo->mem.mem_type == TTM_PL_VRAM &&
1008 (offset + size) > adev->mc.visible_vram_size)
1009 return -EINVAL;
1010
1011 return 0;
1012}
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1023 bool shared)
1024{
1025 struct reservation_object *resv = bo->tbo.resv;
1026
1027 if (shared)
1028 reservation_object_add_shared_fence(resv, fence);
1029 else
1030 reservation_object_add_excl_fence(resv, fence);
1031}
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1043{
1044 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
1045 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
1046 !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem));
1047 WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
1048 !bo->pin_count);
1049 WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
1050 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
1051 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1052
1053 return bo->tbo.offset;
1054}
1055
