1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/dma-mapping.h>
34#include <linux/iommu.h>
35#include <linux/hmm.h>
36#include <linux/pagemap.h>
37#include <linux/sched/task.h>
38#include <linux/seq_file.h>
39#include <linux/slab.h>
40#include <linux/swap.h>
41#include <linux/swiotlb.h>
42
43#include <drm/ttm/ttm_bo_api.h>
44#include <drm/ttm/ttm_bo_driver.h>
45#include <drm/ttm/ttm_placement.h>
46#include <drm/ttm/ttm_module.h>
47#include <drm/ttm/ttm_page_alloc.h>
48
49#include <drm/drm_debugfs.h>
50#include <drm/amdgpu_drm.h>
51
52#include "amdgpu.h"
53#include "amdgpu_object.h"
54#include "amdgpu_trace.h"
55#include "amdgpu_amdkfd.h"
56#include "amdgpu_sdma.h"
57#include "bif/bif_4_1_d.h"
58
59static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
60 struct ttm_mem_reg *mem, unsigned num_pages,
61 uint64_t offset, unsigned window,
62 struct amdgpu_ring *ring,
63 uint64_t *addr);
64
65static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
66static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
67
68static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
69{
70 return 0;
71}
72
73
74
75
76
77
78
79
80
81
82
83
84static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
85 struct ttm_mem_type_manager *man)
86{
87 struct amdgpu_device *adev;
88
89 adev = amdgpu_ttm_adev(bdev);
90
91 switch (type) {
92 case TTM_PL_SYSTEM:
93
94 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
95 man->available_caching = TTM_PL_MASK_CACHING;
96 man->default_caching = TTM_PL_FLAG_CACHED;
97 break;
98 case TTM_PL_TT:
99
100 man->func = &amdgpu_gtt_mgr_func;
101 man->gpu_offset = adev->gmc.gart_start;
102 man->available_caching = TTM_PL_MASK_CACHING;
103 man->default_caching = TTM_PL_FLAG_CACHED;
104 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
105 break;
106 case TTM_PL_VRAM:
107
108 man->func = &amdgpu_vram_mgr_func;
109 man->gpu_offset = adev->gmc.vram_start;
110 man->flags = TTM_MEMTYPE_FLAG_FIXED |
111 TTM_MEMTYPE_FLAG_MAPPABLE;
112 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
113 man->default_caching = TTM_PL_FLAG_WC;
114 break;
115 case AMDGPU_PL_GDS:
116 case AMDGPU_PL_GWS:
117 case AMDGPU_PL_OA:
118
119 man->func = &ttm_bo_manager_func;
120 man->gpu_offset = 0;
121 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
122 man->available_caching = TTM_PL_FLAG_UNCACHED;
123 man->default_caching = TTM_PL_FLAG_UNCACHED;
124 break;
125 default:
126 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
127 return -EINVAL;
128 }
129 return 0;
130}
131
132
133
134
135
136
137
138
139
140static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
141 struct ttm_placement *placement)
142{
143 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
144 struct amdgpu_bo *abo;
145 static const struct ttm_place placements = {
146 .fpfn = 0,
147 .lpfn = 0,
148 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
149 };
150
151
152 if (bo->type == ttm_bo_type_sg) {
153 placement->num_placement = 0;
154 placement->num_busy_placement = 0;
155 return;
156 }
157
158
159 if (!amdgpu_bo_is_amdgpu_bo(bo)) {
160 placement->placement = &placements;
161 placement->busy_placement = &placements;
162 placement->num_placement = 1;
163 placement->num_busy_placement = 1;
164 return;
165 }
166
167 abo = ttm_to_amdgpu_bo(bo);
168 switch (bo->mem.mem_type) {
169 case AMDGPU_PL_GDS:
170 case AMDGPU_PL_GWS:
171 case AMDGPU_PL_OA:
172 placement->num_placement = 0;
173 placement->num_busy_placement = 0;
174 return;
175
176 case TTM_PL_VRAM:
177 if (!adev->mman.buffer_funcs_enabled) {
178
179 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
180 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
181 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
182 amdgpu_bo_in_cpu_visible_vram(abo)) {
183
184
185
186
187
188
189 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
190 AMDGPU_GEM_DOMAIN_GTT);
191 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
192 abo->placements[0].lpfn = 0;
193 abo->placement.busy_placement = &abo->placements[1];
194 abo->placement.num_busy_placement = 1;
195 } else {
196
197 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
198 }
199 break;
200 case TTM_PL_TT:
201 default:
202 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
203 break;
204 }
205 *placement = abo->placement;
206}
207
208
209
210
211
212
213
214
215
216
217static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
218{
219 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
220
221
222
223
224
225 if (abo->kfd_bo)
226 return 0;
227
228 if (amdgpu_ttm_tt_get_usermm(bo->ttm))
229 return -EPERM;
230 return drm_vma_node_verify_access(&abo->gem_base.vma_node,
231 filp->private_data);
232}
233
234
235
236
237
238
239
240
241
242static void amdgpu_move_null(struct ttm_buffer_object *bo,
243 struct ttm_mem_reg *new_mem)
244{
245 struct ttm_mem_reg *old_mem = &bo->mem;
246
247 BUG_ON(old_mem->mm_node != NULL);
248 *old_mem = *new_mem;
249 new_mem->mm_node = NULL;
250}
251
252
253
254
255
256
257
258
259
260static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
261 struct drm_mm_node *mm_node,
262 struct ttm_mem_reg *mem)
263{
264 uint64_t addr = 0;
265
266 if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) {
267 addr = mm_node->start << PAGE_SHIFT;
268 addr += bo->bdev->man[mem->mem_type].gpu_offset;
269 }
270 return addr;
271}
272
273
274
275
276
277
278
279
280
281static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
282 unsigned long *offset)
283{
284 struct drm_mm_node *mm_node = mem->mm_node;
285
286 while (*offset >= (mm_node->size << PAGE_SHIFT)) {
287 *offset -= (mm_node->size << PAGE_SHIFT);
288 ++mm_node;
289 }
290 return mm_node;
291}
292
293
294
295
296
297
298
299
300
301
302int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
303 struct amdgpu_copy_mem *src,
304 struct amdgpu_copy_mem *dst,
305 uint64_t size,
306 struct reservation_object *resv,
307 struct dma_fence **f)
308{
309 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
310 struct drm_mm_node *src_mm, *dst_mm;
311 uint64_t src_node_start, dst_node_start, src_node_size,
312 dst_node_size, src_page_offset, dst_page_offset;
313 struct dma_fence *fence = NULL;
314 int r = 0;
315 const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
316 AMDGPU_GPU_PAGE_SIZE);
317
318 if (!adev->mman.buffer_funcs_enabled) {
319 DRM_ERROR("Trying to move memory with ring turned off.\n");
320 return -EINVAL;
321 }
322
323 src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
324 src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
325 src->offset;
326 src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
327 src_page_offset = src_node_start & (PAGE_SIZE - 1);
328
329 dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
330 dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
331 dst->offset;
332 dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
333 dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
334
335 mutex_lock(&adev->mman.gtt_window_lock);
336
337 while (size) {
338 unsigned long cur_size;
339 uint64_t from = src_node_start, to = dst_node_start;
340 struct dma_fence *next;
341
342
343
344
345 cur_size = min3(min(src_node_size, dst_node_size), size,
346 GTT_MAX_BYTES);
347 if (cur_size + src_page_offset > GTT_MAX_BYTES ||
348 cur_size + dst_page_offset > GTT_MAX_BYTES)
349 cur_size -= max(src_page_offset, dst_page_offset);
350
351
352
353
354 if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) {
355 r = amdgpu_map_buffer(src->bo, src->mem,
356 PFN_UP(cur_size + src_page_offset),
357 src_node_start, 0, ring,
358 &from);
359 if (r)
360 goto error;
361
362
363
364 from += src_page_offset;
365 }
366
367 if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) {
368 r = amdgpu_map_buffer(dst->bo, dst->mem,
369 PFN_UP(cur_size + dst_page_offset),
370 dst_node_start, 1, ring,
371 &to);
372 if (r)
373 goto error;
374 to += dst_page_offset;
375 }
376
377 r = amdgpu_copy_buffer(ring, from, to, cur_size,
378 resv, &next, false, true);
379 if (r)
380 goto error;
381
382 dma_fence_put(fence);
383 fence = next;
384
385 size -= cur_size;
386 if (!size)
387 break;
388
389 src_node_size -= cur_size;
390 if (!src_node_size) {
391 src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
392 src->mem);
393 src_node_size = (src_mm->size << PAGE_SHIFT);
394 src_page_offset = 0;
395 } else {
396 src_node_start += cur_size;
397 src_page_offset = src_node_start & (PAGE_SIZE - 1);
398 }
399 dst_node_size -= cur_size;
400 if (!dst_node_size) {
401 dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
402 dst->mem);
403 dst_node_size = (dst_mm->size << PAGE_SHIFT);
404 dst_page_offset = 0;
405 } else {
406 dst_node_start += cur_size;
407 dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
408 }
409 }
410error:
411 mutex_unlock(&adev->mman.gtt_window_lock);
412 if (f)
413 *f = dma_fence_get(fence);
414 dma_fence_put(fence);
415 return r;
416}
417
418
419
420
421
422
423
424static int amdgpu_move_blit(struct ttm_buffer_object *bo,
425 bool evict, bool no_wait_gpu,
426 struct ttm_mem_reg *new_mem,
427 struct ttm_mem_reg *old_mem)
428{
429 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
430 struct amdgpu_copy_mem src, dst;
431 struct dma_fence *fence = NULL;
432 int r;
433
434 src.bo = bo;
435 dst.bo = bo;
436 src.mem = old_mem;
437 dst.mem = new_mem;
438 src.offset = 0;
439 dst.offset = 0;
440
441 r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
442 new_mem->num_pages << PAGE_SHIFT,
443 bo->resv, &fence);
444 if (r)
445 goto error;
446
447
448 if (bo->type == ttm_bo_type_kernel)
449 r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem);
450 else
451 r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
452 dma_fence_put(fence);
453 return r;
454
455error:
456 if (fence)
457 dma_fence_wait(fence, false);
458 dma_fence_put(fence);
459 return r;
460}
461
462
463
464
465
466
467static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
468 struct ttm_operation_ctx *ctx,
469 struct ttm_mem_reg *new_mem)
470{
471 struct amdgpu_device *adev;
472 struct ttm_mem_reg *old_mem = &bo->mem;
473 struct ttm_mem_reg tmp_mem;
474 struct ttm_place placements;
475 struct ttm_placement placement;
476 int r;
477
478 adev = amdgpu_ttm_adev(bo->bdev);
479
480
481 tmp_mem = *new_mem;
482 tmp_mem.mm_node = NULL;
483 placement.num_placement = 1;
484 placement.placement = &placements;
485 placement.num_busy_placement = 1;
486 placement.busy_placement = &placements;
487 placements.fpfn = 0;
488 placements.lpfn = 0;
489 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
490 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
491 if (unlikely(r)) {
492 pr_err("Failed to find GTT space for blit from VRAM\n");
493 return r;
494 }
495
496
497 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
498 if (unlikely(r)) {
499 goto out_cleanup;
500 }
501
502
503 r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
504 if (unlikely(r)) {
505 goto out_cleanup;
506 }
507
508
509 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem);
510 if (unlikely(r)) {
511 goto out_cleanup;
512 }
513
514
515 r = ttm_bo_move_ttm(bo, ctx, new_mem);
516out_cleanup:
517 ttm_bo_mem_put(bo, &tmp_mem);
518 return r;
519}
520
521
522
523
524
525
526static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
527 struct ttm_operation_ctx *ctx,
528 struct ttm_mem_reg *new_mem)
529{
530 struct amdgpu_device *adev;
531 struct ttm_mem_reg *old_mem = &bo->mem;
532 struct ttm_mem_reg tmp_mem;
533 struct ttm_placement placement;
534 struct ttm_place placements;
535 int r;
536
537 adev = amdgpu_ttm_adev(bo->bdev);
538
539
540 tmp_mem = *new_mem;
541 tmp_mem.mm_node = NULL;
542 placement.num_placement = 1;
543 placement.placement = &placements;
544 placement.num_busy_placement = 1;
545 placement.busy_placement = &placements;
546 placements.fpfn = 0;
547 placements.lpfn = 0;
548 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
549 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
550 if (unlikely(r)) {
551 pr_err("Failed to find GTT space for blit to VRAM\n");
552 return r;
553 }
554
555
556 r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
557 if (unlikely(r)) {
558 goto out_cleanup;
559 }
560
561
562 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem);
563 if (unlikely(r)) {
564 goto out_cleanup;
565 }
566out_cleanup:
567 ttm_bo_mem_put(bo, &tmp_mem);
568 return r;
569}
570
571
572
573
574
575
576static bool amdgpu_mem_visible(struct amdgpu_device *adev,
577 struct ttm_mem_reg *mem)
578{
579 struct drm_mm_node *nodes = mem->mm_node;
580
581 if (mem->mem_type == TTM_PL_SYSTEM ||
582 mem->mem_type == TTM_PL_TT)
583 return true;
584 if (mem->mem_type != TTM_PL_VRAM)
585 return false;
586
587
588 if (nodes->size != mem->num_pages)
589 return false;
590
591 return ((nodes->start + nodes->size) << PAGE_SHIFT)
592 <= adev->gmc.visible_vram_size;
593}
594
595
596
597
598
599
600static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
601 struct ttm_operation_ctx *ctx,
602 struct ttm_mem_reg *new_mem)
603{
604 struct amdgpu_device *adev;
605 struct amdgpu_bo *abo;
606 struct ttm_mem_reg *old_mem = &bo->mem;
607 int r;
608
609
610 abo = ttm_to_amdgpu_bo(bo);
611 if (WARN_ON_ONCE(abo->pin_count > 0))
612 return -EINVAL;
613
614 adev = amdgpu_ttm_adev(bo->bdev);
615
616 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
617 amdgpu_move_null(bo, new_mem);
618 return 0;
619 }
620 if ((old_mem->mem_type == TTM_PL_TT &&
621 new_mem->mem_type == TTM_PL_SYSTEM) ||
622 (old_mem->mem_type == TTM_PL_SYSTEM &&
623 new_mem->mem_type == TTM_PL_TT)) {
624
625 amdgpu_move_null(bo, new_mem);
626 return 0;
627 }
628 if (old_mem->mem_type == AMDGPU_PL_GDS ||
629 old_mem->mem_type == AMDGPU_PL_GWS ||
630 old_mem->mem_type == AMDGPU_PL_OA ||
631 new_mem->mem_type == AMDGPU_PL_GDS ||
632 new_mem->mem_type == AMDGPU_PL_GWS ||
633 new_mem->mem_type == AMDGPU_PL_OA) {
634
635 amdgpu_move_null(bo, new_mem);
636 return 0;
637 }
638
639 if (!adev->mman.buffer_funcs_enabled) {
640 r = -ENODEV;
641 goto memcpy;
642 }
643
644 if (old_mem->mem_type == TTM_PL_VRAM &&
645 new_mem->mem_type == TTM_PL_SYSTEM) {
646 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
647 } else if (old_mem->mem_type == TTM_PL_SYSTEM &&
648 new_mem->mem_type == TTM_PL_VRAM) {
649 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
650 } else {
651 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
652 new_mem, old_mem);
653 }
654
655 if (r) {
656memcpy:
657
658 if (!amdgpu_mem_visible(adev, old_mem) ||
659 !amdgpu_mem_visible(adev, new_mem)) {
660 pr_err("Move buffer fallback to memcpy unavailable\n");
661 return r;
662 }
663
664 r = ttm_bo_move_memcpy(bo, ctx, new_mem);
665 if (r)
666 return r;
667 }
668
669 if (bo->type == ttm_bo_type_device &&
670 new_mem->mem_type == TTM_PL_VRAM &&
671 old_mem->mem_type != TTM_PL_VRAM) {
672
673
674
675 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
676 }
677
678
679 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
680 return 0;
681}
682
683
684
685
686
687
688static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
689{
690 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
691 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
692 struct drm_mm_node *mm_node = mem->mm_node;
693
694 mem->bus.addr = NULL;
695 mem->bus.offset = 0;
696 mem->bus.size = mem->num_pages << PAGE_SHIFT;
697 mem->bus.base = 0;
698 mem->bus.is_iomem = false;
699 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
700 return -EINVAL;
701 switch (mem->mem_type) {
702 case TTM_PL_SYSTEM:
703
704 return 0;
705 case TTM_PL_TT:
706 break;
707 case TTM_PL_VRAM:
708 mem->bus.offset = mem->start << PAGE_SHIFT;
709
710 if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
711 return -EINVAL;
712
713
714
715
716 if (adev->mman.aper_base_kaddr &&
717 (mm_node->size == mem->num_pages))
718 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
719 mem->bus.offset;
720
721 mem->bus.base = adev->gmc.aper_base;
722 mem->bus.is_iomem = true;
723 break;
724 default:
725 return -EINVAL;
726 }
727 return 0;
728}
729
730static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
731{
732}
733
734static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
735 unsigned long page_offset)
736{
737 struct drm_mm_node *mm;
738 unsigned long offset = (page_offset << PAGE_SHIFT);
739
740 mm = amdgpu_find_mm_node(&bo->mem, &offset);
741 return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
742 (offset >> PAGE_SHIFT);
743}
744
745
746
747
748struct amdgpu_ttm_tt {
749 struct ttm_dma_tt ttm;
750 u64 offset;
751 uint64_t userptr;
752 struct task_struct *usertask;
753 uint32_t userflags;
754#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
755 struct hmm_range *range;
756#endif
757};
758
759
760
761
762
763
764
765
766#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
767
768#define MAX_RETRY_HMM_RANGE_FAULT 16
769
770int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
771{
772 struct hmm_mirror *mirror = bo->mn ? &bo->mn->mirror : NULL;
773 struct ttm_tt *ttm = bo->tbo.ttm;
774 struct amdgpu_ttm_tt *gtt = (void *)ttm;
775 struct mm_struct *mm = gtt->usertask->mm;
776 unsigned long start = gtt->userptr;
777 struct vm_area_struct *vma;
778 struct hmm_range *range;
779 unsigned long i;
780 uint64_t *pfns;
781 int retry = 0;
782 int r = 0;
783
784 if (!mm)
785 return -ESRCH;
786
787 if (unlikely(!mirror)) {
788 DRM_DEBUG_DRIVER("Failed to get hmm_mirror\n");
789 r = -EFAULT;
790 goto out;
791 }
792
793 vma = find_vma(mm, start);
794 if (unlikely(!vma || start < vma->vm_start)) {
795 r = -EFAULT;
796 goto out;
797 }
798 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
799 vma->vm_file)) {
800 r = -EPERM;
801 goto out;
802 }
803
804 range = kzalloc(sizeof(*range), GFP_KERNEL);
805 if (unlikely(!range)) {
806 r = -ENOMEM;
807 goto out;
808 }
809
810 pfns = kvmalloc_array(ttm->num_pages, sizeof(*pfns), GFP_KERNEL);
811 if (unlikely(!pfns)) {
812 r = -ENOMEM;
813 goto out_free_ranges;
814 }
815
816 amdgpu_hmm_init_range(range);
817 range->default_flags = range->flags[HMM_PFN_VALID];
818 range->default_flags |= amdgpu_ttm_tt_is_readonly(ttm) ?
819 0 : range->flags[HMM_PFN_WRITE];
820 range->pfn_flags_mask = 0;
821 range->pfns = pfns;
822 hmm_range_register(range, mirror, start,
823 start + ttm->num_pages * PAGE_SIZE, PAGE_SHIFT);
824
825retry:
826
827
828
829
830
831 hmm_range_wait_until_valid(range, HMM_RANGE_DEFAULT_TIMEOUT);
832
833 down_read(&mm->mmap_sem);
834
835 r = hmm_range_fault(range, true);
836 if (unlikely(r < 0)) {
837 if (likely(r == -EAGAIN)) {
838
839
840
841 if (retry++ < MAX_RETRY_HMM_RANGE_FAULT)
842 goto retry;
843 else
844 pr_err("Retry hmm fault too many times\n");
845 }
846
847 goto out_up_read;
848 }
849
850 up_read(&mm->mmap_sem);
851
852 for (i = 0; i < ttm->num_pages; i++) {
853 pages[i] = hmm_device_entry_to_page(range, pfns[i]);
854 if (unlikely(!pages[i])) {
855 pr_err("Page fault failed for pfn[%lu] = 0x%llx\n",
856 i, pfns[i]);
857 r = -ENOMEM;
858
859 goto out_free_pfns;
860 }
861 }
862
863 gtt->range = range;
864
865 return 0;
866
867out_up_read:
868 if (likely(r != -EAGAIN))
869 up_read(&mm->mmap_sem);
870out_free_pfns:
871 hmm_range_unregister(range);
872 kvfree(pfns);
873out_free_ranges:
874 kfree(range);
875out:
876 return r;
877}
878
879
880
881
882
883
884
885bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm)
886{
887 struct amdgpu_ttm_tt *gtt = (void *)ttm;
888 bool r = false;
889
890 if (!gtt || !gtt->userptr)
891 return false;
892
893 DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%lx\n",
894 gtt->userptr, ttm->num_pages);
895
896 WARN_ONCE(!gtt->range || !gtt->range->pfns,
897 "No user pages to check\n");
898
899 if (gtt->range) {
900 r = hmm_range_valid(gtt->range);
901 hmm_range_unregister(gtt->range);
902
903 kvfree(gtt->range->pfns);
904 kfree(gtt->range);
905 gtt->range = NULL;
906 }
907
908 return r;
909}
910#endif
911
912
913
914
915
916
917
918
919void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
920{
921 unsigned long i;
922
923 for (i = 0; i < ttm->num_pages; ++i)
924 ttm->pages[i] = pages ? pages[i] : NULL;
925}
926
927
928
929
930
931
932static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
933{
934 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
935 struct amdgpu_ttm_tt *gtt = (void *)ttm;
936 unsigned nents;
937 int r;
938
939 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
940 enum dma_data_direction direction = write ?
941 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
942
943
944 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
945 ttm->num_pages << PAGE_SHIFT,
946 GFP_KERNEL);
947 if (r)
948 goto release_sg;
949
950
951 r = -ENOMEM;
952 nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
953 if (nents != ttm->sg->nents)
954 goto release_sg;
955
956
957 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
958 gtt->ttm.dma_address, ttm->num_pages);
959
960 return 0;
961
962release_sg:
963 kfree(ttm->sg);
964 return r;
965}
966
967
968
969
970static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
971{
972 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
973 struct amdgpu_ttm_tt *gtt = (void *)ttm;
974
975 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
976 enum dma_data_direction direction = write ?
977 DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
978
979
980 if (!ttm->sg->sgl)
981 return;
982
983
984 dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
985
986 sg_free_table(ttm->sg);
987
988#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
989 if (gtt->range &&
990 ttm->pages[0] == hmm_device_entry_to_page(gtt->range,
991 gtt->range->pfns[0]))
992 WARN_ONCE(1, "Missing get_user_page_done\n");
993#endif
994}
995
996int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
997 struct ttm_buffer_object *tbo,
998 uint64_t flags)
999{
1000 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
1001 struct ttm_tt *ttm = tbo->ttm;
1002 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1003 int r;
1004
1005 if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
1006 uint64_t page_idx = 1;
1007
1008 r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
1009 ttm->pages, gtt->ttm.dma_address, flags);
1010 if (r)
1011 goto gart_bind_fail;
1012
1013
1014 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1015 flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC);
1016
1017 r = amdgpu_gart_bind(adev,
1018 gtt->offset + (page_idx << PAGE_SHIFT),
1019 ttm->num_pages - page_idx,
1020 &ttm->pages[page_idx],
1021 &(gtt->ttm.dma_address[page_idx]), flags);
1022 } else {
1023 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1024 ttm->pages, gtt->ttm.dma_address, flags);
1025 }
1026
1027gart_bind_fail:
1028 if (r)
1029 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1030 ttm->num_pages, gtt->offset);
1031
1032 return r;
1033}
1034
1035
1036
1037
1038
1039
1040
1041static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
1042 struct ttm_mem_reg *bo_mem)
1043{
1044 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1045 struct amdgpu_ttm_tt *gtt = (void*)ttm;
1046 uint64_t flags;
1047 int r = 0;
1048
1049 if (gtt->userptr) {
1050 r = amdgpu_ttm_tt_pin_userptr(ttm);
1051 if (r) {
1052 DRM_ERROR("failed to pin userptr\n");
1053 return r;
1054 }
1055 }
1056 if (!ttm->num_pages) {
1057 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
1058 ttm->num_pages, bo_mem, ttm);
1059 }
1060
1061 if (bo_mem->mem_type == AMDGPU_PL_GDS ||
1062 bo_mem->mem_type == AMDGPU_PL_GWS ||
1063 bo_mem->mem_type == AMDGPU_PL_OA)
1064 return -EINVAL;
1065
1066 if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
1067 gtt->offset = AMDGPU_BO_INVALID_OFFSET;
1068 return 0;
1069 }
1070
1071
1072 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
1073
1074
1075 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
1076 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1077 ttm->pages, gtt->ttm.dma_address, flags);
1078
1079 if (r)
1080 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
1081 ttm->num_pages, gtt->offset);
1082 return r;
1083}
1084
1085
1086
1087
1088int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1089{
1090 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1091 struct ttm_operation_ctx ctx = { false, false };
1092 struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
1093 struct ttm_mem_reg tmp;
1094 struct ttm_placement placement;
1095 struct ttm_place placements;
1096 uint64_t addr, flags;
1097 int r;
1098
1099 if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
1100 return 0;
1101
1102 addr = amdgpu_gmc_agp_addr(bo);
1103 if (addr != AMDGPU_BO_INVALID_OFFSET) {
1104 bo->mem.start = addr >> PAGE_SHIFT;
1105 } else {
1106
1107
1108 tmp = bo->mem;
1109 tmp.mm_node = NULL;
1110 placement.num_placement = 1;
1111 placement.placement = &placements;
1112 placement.num_busy_placement = 1;
1113 placement.busy_placement = &placements;
1114 placements.fpfn = 0;
1115 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1116 placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
1117 TTM_PL_FLAG_TT;
1118
1119 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1120 if (unlikely(r))
1121 return r;
1122
1123
1124 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1125
1126
1127 gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1128 r = amdgpu_ttm_gart_bind(adev, bo, flags);
1129 if (unlikely(r)) {
1130 ttm_bo_mem_put(bo, &tmp);
1131 return r;
1132 }
1133
1134 ttm_bo_mem_put(bo, &bo->mem);
1135 bo->mem = tmp;
1136 }
1137
1138 bo->offset = (bo->mem.start << PAGE_SHIFT) +
1139 bo->bdev->man[bo->mem.mem_type].gpu_offset;
1140
1141 return 0;
1142}
1143
1144
1145
1146
1147
1148
1149
1150int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1151{
1152 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1153 uint64_t flags;
1154 int r;
1155
1156 if (!tbo->ttm)
1157 return 0;
1158
1159 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
1160 r = amdgpu_ttm_gart_bind(adev, tbo, flags);
1161
1162 return r;
1163}
1164
1165
1166
1167
1168
1169
1170
1171static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
1172{
1173 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1174 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1175 int r;
1176
1177
1178 if (gtt->userptr)
1179 amdgpu_ttm_tt_unpin_userptr(ttm);
1180
1181 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1182 return 0;
1183
1184
1185 r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1186 if (r)
1187 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
1188 gtt->ttm.ttm.num_pages, gtt->offset);
1189 return r;
1190}
1191
1192static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
1193{
1194 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1195
1196 if (gtt->usertask)
1197 put_task_struct(gtt->usertask);
1198
1199 ttm_dma_tt_fini(>t->ttm);
1200 kfree(gtt);
1201}
1202
1203static struct ttm_backend_func amdgpu_backend_func = {
1204 .bind = &amdgpu_ttm_backend_bind,
1205 .unbind = &amdgpu_ttm_backend_unbind,
1206 .destroy = &amdgpu_ttm_backend_destroy,
1207};
1208
1209
1210
1211
1212
1213
1214
1215
1216static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
1217 uint32_t page_flags)
1218{
1219 struct amdgpu_device *adev;
1220 struct amdgpu_ttm_tt *gtt;
1221
1222 adev = amdgpu_ttm_adev(bo->bdev);
1223
1224 gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
1225 if (gtt == NULL) {
1226 return NULL;
1227 }
1228 gtt->ttm.ttm.func = &amdgpu_backend_func;
1229
1230
1231 if (ttm_sg_tt_init(>t->ttm, bo, page_flags)) {
1232 kfree(gtt);
1233 return NULL;
1234 }
1235 return >t->ttm.ttm;
1236}
1237
1238
1239
1240
1241
1242
1243
1244static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
1245 struct ttm_operation_ctx *ctx)
1246{
1247 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
1248 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1249 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1250
1251
1252 if (gtt && gtt->userptr) {
1253 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
1254 if (!ttm->sg)
1255 return -ENOMEM;
1256
1257 ttm->page_flags |= TTM_PAGE_FLAG_SG;
1258 ttm->state = tt_unbound;
1259 return 0;
1260 }
1261
1262 if (slave && ttm->sg) {
1263 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1264 gtt->ttm.dma_address,
1265 ttm->num_pages);
1266 ttm->state = tt_unbound;
1267 return 0;
1268 }
1269
1270#ifdef CONFIG_SWIOTLB
1271 if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1272 return ttm_dma_populate(>t->ttm, adev->dev, ctx);
1273 }
1274#endif
1275
1276
1277
1278 return ttm_populate_and_map_pages(adev->dev, >t->ttm, ctx);
1279}
1280
1281
1282
1283
1284
1285
1286
1287static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
1288{
1289 struct amdgpu_device *adev;
1290 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1291 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1292
1293 if (gtt && gtt->userptr) {
1294 amdgpu_ttm_tt_set_user_pages(ttm, NULL);
1295 kfree(ttm->sg);
1296 ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
1297 return;
1298 }
1299
1300 if (slave)
1301 return;
1302
1303 adev = amdgpu_ttm_adev(ttm->bdev);
1304
1305#ifdef CONFIG_SWIOTLB
1306 if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1307 ttm_dma_unpopulate(>t->ttm, adev->dev);
1308 return;
1309 }
1310#endif
1311
1312
1313 ttm_unmap_and_unpopulate_pages(adev->dev, >t->ttm);
1314}
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
1328 uint32_t flags)
1329{
1330 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1331
1332 if (gtt == NULL)
1333 return -EINVAL;
1334
1335 gtt->userptr = addr;
1336 gtt->userflags = flags;
1337
1338 if (gtt->usertask)
1339 put_task_struct(gtt->usertask);
1340 gtt->usertask = current->group_leader;
1341 get_task_struct(gtt->usertask);
1342
1343 return 0;
1344}
1345
1346
1347
1348
1349struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
1350{
1351 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1352
1353 if (gtt == NULL)
1354 return NULL;
1355
1356 if (gtt->usertask == NULL)
1357 return NULL;
1358
1359 return gtt->usertask->mm;
1360}
1361
1362
1363
1364
1365
1366
1367bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
1368 unsigned long end)
1369{
1370 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1371 unsigned long size;
1372
1373 if (gtt == NULL || !gtt->userptr)
1374 return false;
1375
1376
1377
1378
1379 size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
1380 if (gtt->userptr > end || gtt->userptr + size <= start)
1381 return false;
1382
1383 return true;
1384}
1385
1386
1387
1388
1389bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
1390{
1391 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1392
1393 if (gtt == NULL || !gtt->userptr)
1394 return false;
1395
1396 return true;
1397}
1398
1399
1400
1401
1402bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
1403{
1404 struct amdgpu_ttm_tt *gtt = (void *)ttm;
1405
1406 if (gtt == NULL)
1407 return false;
1408
1409 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
1410}
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
1421{
1422 uint64_t flags = 0;
1423
1424 if (mem && mem->mem_type != TTM_PL_SYSTEM)
1425 flags |= AMDGPU_PTE_VALID;
1426
1427 if (mem && mem->mem_type == TTM_PL_TT) {
1428 flags |= AMDGPU_PTE_SYSTEM;
1429
1430 if (ttm->caching_state == tt_cached)
1431 flags |= AMDGPU_PTE_SNOOPED;
1432 }
1433
1434 return flags;
1435}
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
1446 struct ttm_mem_reg *mem)
1447{
1448 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
1449
1450 flags |= adev->gart.gart_pte_flags;
1451 flags |= AMDGPU_PTE_READABLE;
1452
1453 if (!amdgpu_ttm_tt_is_readonly(ttm))
1454 flags |= AMDGPU_PTE_WRITEABLE;
1455
1456 return flags;
1457}
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
1469 const struct ttm_place *place)
1470{
1471 unsigned long num_pages = bo->mem.num_pages;
1472 struct drm_mm_node *node = bo->mem.mm_node;
1473 struct reservation_object_list *flist;
1474 struct dma_fence *f;
1475 int i;
1476
1477
1478
1479
1480 if (bo->type == ttm_bo_type_kernel &&
1481 !reservation_object_test_signaled_rcu(bo->resv, true))
1482 return false;
1483
1484
1485
1486
1487
1488 flist = reservation_object_get_list(bo->resv);
1489 if (flist) {
1490 for (i = 0; i < flist->shared_count; ++i) {
1491 f = rcu_dereference_protected(flist->shared[i],
1492 reservation_object_held(bo->resv));
1493 if (amdkfd_fence_check_mm(f, current->mm))
1494 return false;
1495 }
1496 }
1497
1498 switch (bo->mem.mem_type) {
1499 case TTM_PL_TT:
1500 return true;
1501
1502 case TTM_PL_VRAM:
1503
1504 while (num_pages) {
1505 if (place->fpfn < (node->start + node->size) &&
1506 !(place->lpfn && place->lpfn <= node->start))
1507 return true;
1508
1509 num_pages -= node->size;
1510 ++node;
1511 }
1512 return false;
1513
1514 default:
1515 break;
1516 }
1517
1518 return ttm_bo_eviction_valuable(bo, place);
1519}
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
1534 unsigned long offset,
1535 void *buf, int len, int write)
1536{
1537 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1538 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1539 struct drm_mm_node *nodes;
1540 uint32_t value = 0;
1541 int ret = 0;
1542 uint64_t pos;
1543 unsigned long flags;
1544
1545 if (bo->mem.mem_type != TTM_PL_VRAM)
1546 return -EIO;
1547
1548 nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
1549 pos = (nodes->start << PAGE_SHIFT) + offset;
1550
1551 while (len && pos < adev->gmc.mc_vram_size) {
1552 uint64_t aligned_pos = pos & ~(uint64_t)3;
1553 uint32_t bytes = 4 - (pos & 3);
1554 uint32_t shift = (pos & 3) * 8;
1555 uint32_t mask = 0xffffffff << shift;
1556
1557 if (len < bytes) {
1558 mask &= 0xffffffff >> (bytes - len) * 8;
1559 bytes = len;
1560 }
1561
1562 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1563 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
1564 WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1565 if (!write || mask != 0xffffffff)
1566 value = RREG32_NO_KIQ(mmMM_DATA);
1567 if (write) {
1568 value &= ~mask;
1569 value |= (*(uint32_t *)buf << shift) & mask;
1570 WREG32_NO_KIQ(mmMM_DATA, value);
1571 }
1572 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
1573 if (!write) {
1574 value = (value & mask) >> shift;
1575 memcpy(buf, &value, bytes);
1576 }
1577
1578 ret += bytes;
1579 buf = (uint8_t *)buf + bytes;
1580 pos += bytes;
1581 len -= bytes;
1582 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
1583 ++nodes;
1584 pos = (nodes->start << PAGE_SHIFT);
1585 }
1586 }
1587
1588 return ret;
1589}
1590
1591static struct ttm_bo_driver amdgpu_bo_driver = {
1592 .ttm_tt_create = &amdgpu_ttm_tt_create,
1593 .ttm_tt_populate = &amdgpu_ttm_tt_populate,
1594 .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
1595 .invalidate_caches = &amdgpu_invalidate_caches,
1596 .init_mem_type = &amdgpu_init_mem_type,
1597 .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
1598 .evict_flags = &amdgpu_evict_flags,
1599 .move = &amdgpu_bo_move,
1600 .verify_access = &amdgpu_verify_access,
1601 .move_notify = &amdgpu_bo_move_notify,
1602 .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
1603 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
1604 .io_mem_free = &amdgpu_ttm_io_mem_free,
1605 .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1606 .access_memory = &amdgpu_ttm_access_memory,
1607 .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify
1608};
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
1621{
1622 amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
1623 NULL, &adev->fw_vram_usage.va);
1624}
1625
1626
1627
1628
1629
1630
1631
1632
1633static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
1634{
1635 struct ttm_operation_ctx ctx = { false, false };
1636 struct amdgpu_bo_param bp;
1637 int r = 0;
1638 int i;
1639 u64 vram_size = adev->gmc.visible_vram_size;
1640 u64 offset = adev->fw_vram_usage.start_offset;
1641 u64 size = adev->fw_vram_usage.size;
1642 struct amdgpu_bo *bo;
1643
1644 memset(&bp, 0, sizeof(bp));
1645 bp.size = adev->fw_vram_usage.size;
1646 bp.byte_align = PAGE_SIZE;
1647 bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
1648 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1649 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1650 bp.type = ttm_bo_type_kernel;
1651 bp.resv = NULL;
1652 adev->fw_vram_usage.va = NULL;
1653 adev->fw_vram_usage.reserved_bo = NULL;
1654
1655 if (adev->fw_vram_usage.size > 0 &&
1656 adev->fw_vram_usage.size <= vram_size) {
1657
1658 r = amdgpu_bo_create(adev, &bp,
1659 &adev->fw_vram_usage.reserved_bo);
1660 if (r)
1661 goto error_create;
1662
1663 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
1664 if (r)
1665 goto error_reserve;
1666
1667
1668
1669
1670 bo = adev->fw_vram_usage.reserved_bo;
1671 offset = ALIGN(offset, PAGE_SIZE);
1672 for (i = 0; i < bo->placement.num_placement; ++i) {
1673 bo->placements[i].fpfn = offset >> PAGE_SHIFT;
1674 bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
1675 }
1676
1677 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
1678 r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
1679 &bo->tbo.mem, &ctx);
1680 if (r)
1681 goto error_pin;
1682
1683 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
1684 AMDGPU_GEM_DOMAIN_VRAM,
1685 adev->fw_vram_usage.start_offset,
1686 (adev->fw_vram_usage.start_offset +
1687 adev->fw_vram_usage.size));
1688 if (r)
1689 goto error_pin;
1690 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
1691 &adev->fw_vram_usage.va);
1692 if (r)
1693 goto error_kmap;
1694
1695 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1696 }
1697 return r;
1698
1699error_kmap:
1700 amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
1701error_pin:
1702 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
1703error_reserve:
1704 amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
1705error_create:
1706 adev->fw_vram_usage.va = NULL;
1707 adev->fw_vram_usage.reserved_bo = NULL;
1708 return r;
1709}
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719int amdgpu_ttm_init(struct amdgpu_device *adev)
1720{
1721 uint64_t gtt_size;
1722 int r;
1723 u64 vis_vram_limit;
1724
1725 mutex_init(&adev->mman.gtt_window_lock);
1726
1727
1728 r = ttm_bo_device_init(&adev->mman.bdev,
1729 &amdgpu_bo_driver,
1730 adev->ddev->anon_inode->i_mapping,
1731 adev->need_dma32);
1732 if (r) {
1733 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
1734 return r;
1735 }
1736 adev->mman.initialized = true;
1737
1738
1739 adev->mman.bdev.no_retry = true;
1740
1741
1742 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
1743 adev->gmc.real_vram_size >> PAGE_SHIFT);
1744 if (r) {
1745 DRM_ERROR("Failed initializing VRAM heap.\n");
1746 return r;
1747 }
1748
1749
1750 vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
1751 if (amdgpu_vis_vram_limit > 0 &&
1752 vis_vram_limit <= adev->gmc.visible_vram_size)
1753 adev->gmc.visible_vram_size = vis_vram_limit;
1754
1755
1756 amdgpu_ttm_set_buffer_funcs_status(adev, false);
1757#ifdef CONFIG_64BIT
1758 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
1759 adev->gmc.visible_vram_size);
1760#endif
1761
1762
1763
1764
1765
1766 r = amdgpu_ttm_fw_reserve_vram_init(adev);
1767 if (r) {
1768 return r;
1769 }
1770
1771
1772
1773
1774
1775 r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
1776 AMDGPU_GEM_DOMAIN_VRAM,
1777 &adev->stolen_vga_memory,
1778 NULL, NULL);
1779 if (r)
1780 return r;
1781 DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1782 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1783
1784
1785
1786 if (amdgpu_gtt_size == -1) {
1787 struct sysinfo si;
1788
1789 si_meminfo(&si);
1790 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1791 adev->gmc.mc_vram_size),
1792 ((uint64_t)si.totalram * si.mem_unit * 3/4));
1793 }
1794 else
1795 gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1796
1797
1798 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
1799 if (r) {
1800 DRM_ERROR("Failed initializing GTT heap.\n");
1801 return r;
1802 }
1803 DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1804 (unsigned)(gtt_size / (1024 * 1024)));
1805
1806
1807 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
1808 adev->gds.gds_size);
1809 if (r) {
1810 DRM_ERROR("Failed initializing GDS heap.\n");
1811 return r;
1812 }
1813
1814 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
1815 adev->gds.gws_size);
1816 if (r) {
1817 DRM_ERROR("Failed initializing gws heap.\n");
1818 return r;
1819 }
1820
1821 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
1822 adev->gds.oa_size);
1823 if (r) {
1824 DRM_ERROR("Failed initializing oa heap.\n");
1825 return r;
1826 }
1827
1828
1829 r = amdgpu_ttm_debugfs_init(adev);
1830 if (r) {
1831 DRM_ERROR("Failed to init debugfs\n");
1832 return r;
1833 }
1834 return 0;
1835}
1836
1837
1838
1839
1840void amdgpu_ttm_late_init(struct amdgpu_device *adev)
1841{
1842
1843 amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
1844}
1845
1846
1847
1848
1849void amdgpu_ttm_fini(struct amdgpu_device *adev)
1850{
1851 if (!adev->mman.initialized)
1852 return;
1853
1854 amdgpu_ttm_debugfs_fini(adev);
1855 amdgpu_ttm_fw_reserve_vram_fini(adev);
1856 if (adev->mman.aper_base_kaddr)
1857 iounmap(adev->mman.aper_base_kaddr);
1858 adev->mman.aper_base_kaddr = NULL;
1859
1860 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
1861 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1862 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
1863 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
1864 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
1865 ttm_bo_device_release(&adev->mman.bdev);
1866 adev->mman.initialized = false;
1867 DRM_INFO("amdgpu: ttm finalized\n");
1868}
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
1880{
1881 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
1882 uint64_t size;
1883 int r;
1884
1885 if (!adev->mman.initialized || adev->in_gpu_reset ||
1886 adev->mman.buffer_funcs_enabled == enable)
1887 return;
1888
1889 if (enable) {
1890 struct amdgpu_ring *ring;
1891 struct drm_sched_rq *rq;
1892
1893 ring = adev->mman.buffer_funcs_ring;
1894 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
1895 r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
1896 if (r) {
1897 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
1898 r);
1899 return;
1900 }
1901 } else {
1902 drm_sched_entity_destroy(&adev->mman.entity);
1903 dma_fence_put(man->move);
1904 man->move = NULL;
1905 }
1906
1907
1908 if (enable)
1909 size = adev->gmc.real_vram_size;
1910 else
1911 size = adev->gmc.visible_vram_size;
1912 man->size = size >> PAGE_SHIFT;
1913 adev->mman.buffer_funcs_enabled = enable;
1914}
1915
1916int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
1917{
1918 struct drm_file *file_priv = filp->private_data;
1919 struct amdgpu_device *adev = file_priv->minor->dev->dev_private;
1920
1921 if (adev == NULL)
1922 return -EINVAL;
1923
1924 return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
1925}
1926
1927static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
1928 struct ttm_mem_reg *mem, unsigned num_pages,
1929 uint64_t offset, unsigned window,
1930 struct amdgpu_ring *ring,
1931 uint64_t *addr)
1932{
1933 struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
1934 struct amdgpu_device *adev = ring->adev;
1935 struct ttm_tt *ttm = bo->ttm;
1936 struct amdgpu_job *job;
1937 unsigned num_dw, num_bytes;
1938 dma_addr_t *dma_address;
1939 struct dma_fence *fence;
1940 uint64_t src_addr, dst_addr;
1941 uint64_t flags;
1942 int r;
1943
1944 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
1945 AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
1946
1947 *addr = adev->gmc.gart_start;
1948 *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
1949 AMDGPU_GPU_PAGE_SIZE;
1950
1951 num_dw = adev->mman.buffer_funcs->copy_num_dw;
1952 while (num_dw & 0x7)
1953 num_dw++;
1954
1955 num_bytes = num_pages * 8;
1956
1957 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
1958 if (r)
1959 return r;
1960
1961 src_addr = num_dw * 4;
1962 src_addr += job->ibs[0].gpu_addr;
1963
1964 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
1965 dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
1966 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
1967 dst_addr, num_bytes);
1968
1969 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
1970 WARN_ON(job->ibs[0].length_dw > num_dw);
1971
1972 dma_address = >t->ttm.dma_address[offset >> PAGE_SHIFT];
1973 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
1974 r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
1975 &job->ibs[0].ptr[num_dw]);
1976 if (r)
1977 goto error_free;
1978
1979 r = amdgpu_job_submit(job, &adev->mman.entity,
1980 AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
1981 if (r)
1982 goto error_free;
1983
1984 dma_fence_put(fence);
1985
1986 return r;
1987
1988error_free:
1989 amdgpu_job_free(job);
1990 return r;
1991}
1992
1993int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
1994 uint64_t dst_offset, uint32_t byte_count,
1995 struct reservation_object *resv,
1996 struct dma_fence **fence, bool direct_submit,
1997 bool vm_needs_flush)
1998{
1999 struct amdgpu_device *adev = ring->adev;
2000 struct amdgpu_job *job;
2001
2002 uint32_t max_bytes;
2003 unsigned num_loops, num_dw;
2004 unsigned i;
2005 int r;
2006
2007 if (direct_submit && !ring->sched.ready) {
2008 DRM_ERROR("Trying to move memory with ring turned off.\n");
2009 return -EINVAL;
2010 }
2011
2012 max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
2013 num_loops = DIV_ROUND_UP(byte_count, max_bytes);
2014 num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;
2015
2016
2017 while (num_dw & 0x7)
2018 num_dw++;
2019
2020 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2021 if (r)
2022 return r;
2023
2024 if (vm_needs_flush) {
2025 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
2026 job->vm_needs_flush = true;
2027 }
2028 if (resv) {
2029 r = amdgpu_sync_resv(adev, &job->sync, resv,
2030 AMDGPU_FENCE_OWNER_UNDEFINED,
2031 false);
2032 if (r) {
2033 DRM_ERROR("sync failed (%d).\n", r);
2034 goto error_free;
2035 }
2036 }
2037
2038 for (i = 0; i < num_loops; i++) {
2039 uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
2040
2041 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
2042 dst_offset, cur_size_in_bytes);
2043
2044 src_offset += cur_size_in_bytes;
2045 dst_offset += cur_size_in_bytes;
2046 byte_count -= cur_size_in_bytes;
2047 }
2048
2049 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2050 WARN_ON(job->ibs[0].length_dw > num_dw);
2051 if (direct_submit)
2052 r = amdgpu_job_submit_direct(job, ring, fence);
2053 else
2054 r = amdgpu_job_submit(job, &adev->mman.entity,
2055 AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2056 if (r)
2057 goto error_free;
2058
2059 return r;
2060
2061error_free:
2062 amdgpu_job_free(job);
2063 DRM_ERROR("Error scheduling IBs (%d)\n", r);
2064 return r;
2065}
2066
2067int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2068 uint32_t src_data,
2069 struct reservation_object *resv,
2070 struct dma_fence **fence)
2071{
2072 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2073 uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2074 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2075
2076 struct drm_mm_node *mm_node;
2077 unsigned long num_pages;
2078 unsigned int num_loops, num_dw;
2079
2080 struct amdgpu_job *job;
2081 int r;
2082
2083 if (!adev->mman.buffer_funcs_enabled) {
2084 DRM_ERROR("Trying to clear memory with ring turned off.\n");
2085 return -EINVAL;
2086 }
2087
2088 if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2089 r = amdgpu_ttm_alloc_gart(&bo->tbo);
2090 if (r)
2091 return r;
2092 }
2093
2094 num_pages = bo->tbo.num_pages;
2095 mm_node = bo->tbo.mem.mm_node;
2096 num_loops = 0;
2097 while (num_pages) {
2098 uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2099
2100 num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes);
2101 num_pages -= mm_node->size;
2102 ++mm_node;
2103 }
2104 num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2105
2106
2107 num_dw += 64;
2108
2109 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
2110 if (r)
2111 return r;
2112
2113 if (resv) {
2114 r = amdgpu_sync_resv(adev, &job->sync, resv,
2115 AMDGPU_FENCE_OWNER_UNDEFINED, false);
2116 if (r) {
2117 DRM_ERROR("sync failed (%d).\n", r);
2118 goto error_free;
2119 }
2120 }
2121
2122 num_pages = bo->tbo.num_pages;
2123 mm_node = bo->tbo.mem.mm_node;
2124
2125 while (num_pages) {
2126 uint64_t byte_count = mm_node->size << PAGE_SHIFT;
2127 uint64_t dst_addr;
2128
2129 dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
2130 while (byte_count) {
2131 uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count,
2132 max_bytes);
2133
2134 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
2135 dst_addr, cur_size_in_bytes);
2136
2137 dst_addr += cur_size_in_bytes;
2138 byte_count -= cur_size_in_bytes;
2139 }
2140
2141 num_pages -= mm_node->size;
2142 ++mm_node;
2143 }
2144
2145 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
2146 WARN_ON(job->ibs[0].length_dw > num_dw);
2147 r = amdgpu_job_submit(job, &adev->mman.entity,
2148 AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2149 if (r)
2150 goto error_free;
2151
2152 return 0;
2153
2154error_free:
2155 amdgpu_job_free(job);
2156 return r;
2157}
2158
2159#if defined(CONFIG_DEBUG_FS)
2160
2161static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
2162{
2163 struct drm_info_node *node = (struct drm_info_node *)m->private;
2164 unsigned ttm_pl = (uintptr_t)node->info_ent->data;
2165 struct drm_device *dev = node->minor->dev;
2166 struct amdgpu_device *adev = dev->dev_private;
2167 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
2168 struct drm_printer p = drm_seq_file_printer(m);
2169
2170 man->func->debug(man, &p);
2171 return 0;
2172}
2173
2174static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
2175 {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM},
2176 {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT},
2177 {"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS},
2178 {"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS},
2179 {"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA},
2180 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
2181#ifdef CONFIG_SWIOTLB
2182 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
2183#endif
2184};
2185
2186
2187
2188
2189
2190
2191static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
2192 size_t size, loff_t *pos)
2193{
2194 struct amdgpu_device *adev = file_inode(f)->i_private;
2195 ssize_t result = 0;
2196 int r;
2197
2198 if (size & 0x3 || *pos & 0x3)
2199 return -EINVAL;
2200
2201 if (*pos >= adev->gmc.mc_vram_size)
2202 return -ENXIO;
2203
2204 while (size) {
2205 unsigned long flags;
2206 uint32_t value;
2207
2208 if (*pos >= adev->gmc.mc_vram_size)
2209 return result;
2210
2211 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2212 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2213 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2214 value = RREG32_NO_KIQ(mmMM_DATA);
2215 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2216
2217 r = put_user(value, (uint32_t *)buf);
2218 if (r)
2219 return r;
2220
2221 result += 4;
2222 buf += 4;
2223 *pos += 4;
2224 size -= 4;
2225 }
2226
2227 return result;
2228}
2229
2230
2231
2232
2233
2234
2235static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
2236 size_t size, loff_t *pos)
2237{
2238 struct amdgpu_device *adev = file_inode(f)->i_private;
2239 ssize_t result = 0;
2240 int r;
2241
2242 if (size & 0x3 || *pos & 0x3)
2243 return -EINVAL;
2244
2245 if (*pos >= adev->gmc.mc_vram_size)
2246 return -ENXIO;
2247
2248 while (size) {
2249 unsigned long flags;
2250 uint32_t value;
2251
2252 if (*pos >= adev->gmc.mc_vram_size)
2253 return result;
2254
2255 r = get_user(value, (uint32_t *)buf);
2256 if (r)
2257 return r;
2258
2259 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2260 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
2261 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
2262 WREG32_NO_KIQ(mmMM_DATA, value);
2263 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
2264
2265 result += 4;
2266 buf += 4;
2267 *pos += 4;
2268 size -= 4;
2269 }
2270
2271 return result;
2272}
2273
2274static const struct file_operations amdgpu_ttm_vram_fops = {
2275 .owner = THIS_MODULE,
2276 .read = amdgpu_ttm_vram_read,
2277 .write = amdgpu_ttm_vram_write,
2278 .llseek = default_llseek,
2279};
2280
2281#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2282
2283
2284
2285
2286static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
2287 size_t size, loff_t *pos)
2288{
2289 struct amdgpu_device *adev = file_inode(f)->i_private;
2290 ssize_t result = 0;
2291 int r;
2292
2293 while (size) {
2294 loff_t p = *pos / PAGE_SIZE;
2295 unsigned off = *pos & ~PAGE_MASK;
2296 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
2297 struct page *page;
2298 void *ptr;
2299
2300 if (p >= adev->gart.num_cpu_pages)
2301 return result;
2302
2303 page = adev->gart.pages[p];
2304 if (page) {
2305 ptr = kmap(page);
2306 ptr += off;
2307
2308 r = copy_to_user(buf, ptr, cur_size);
2309 kunmap(adev->gart.pages[p]);
2310 } else
2311 r = clear_user(buf, cur_size);
2312
2313 if (r)
2314 return -EFAULT;
2315
2316 result += cur_size;
2317 buf += cur_size;
2318 *pos += cur_size;
2319 size -= cur_size;
2320 }
2321
2322 return result;
2323}
2324
2325static const struct file_operations amdgpu_ttm_gtt_fops = {
2326 .owner = THIS_MODULE,
2327 .read = amdgpu_ttm_gtt_read,
2328 .llseek = default_llseek
2329};
2330
2331#endif
2332
2333
2334
2335
2336
2337
2338
2339
2340static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
2341 size_t size, loff_t *pos)
2342{
2343 struct amdgpu_device *adev = file_inode(f)->i_private;
2344 struct iommu_domain *dom;
2345 ssize_t result = 0;
2346 int r;
2347
2348
2349 dom = iommu_get_domain_for_dev(adev->dev);
2350
2351 while (size) {
2352 phys_addr_t addr = *pos & PAGE_MASK;
2353 loff_t off = *pos & ~PAGE_MASK;
2354 size_t bytes = PAGE_SIZE - off;
2355 unsigned long pfn;
2356 struct page *p;
2357 void *ptr;
2358
2359 bytes = bytes < size ? bytes : size;
2360
2361
2362
2363
2364
2365 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2366
2367 pfn = addr >> PAGE_SHIFT;
2368 if (!pfn_valid(pfn))
2369 return -EPERM;
2370
2371 p = pfn_to_page(pfn);
2372 if (p->mapping != adev->mman.bdev.dev_mapping)
2373 return -EPERM;
2374
2375 ptr = kmap(p);
2376 r = copy_to_user(buf, ptr + off, bytes);
2377 kunmap(p);
2378 if (r)
2379 return -EFAULT;
2380
2381 size -= bytes;
2382 *pos += bytes;
2383 result += bytes;
2384 }
2385
2386 return result;
2387}
2388
2389
2390
2391
2392
2393
2394
2395
2396static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
2397 size_t size, loff_t *pos)
2398{
2399 struct amdgpu_device *adev = file_inode(f)->i_private;
2400 struct iommu_domain *dom;
2401 ssize_t result = 0;
2402 int r;
2403
2404 dom = iommu_get_domain_for_dev(adev->dev);
2405
2406 while (size) {
2407 phys_addr_t addr = *pos & PAGE_MASK;
2408 loff_t off = *pos & ~PAGE_MASK;
2409 size_t bytes = PAGE_SIZE - off;
2410 unsigned long pfn;
2411 struct page *p;
2412 void *ptr;
2413
2414 bytes = bytes < size ? bytes : size;
2415
2416 addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
2417
2418 pfn = addr >> PAGE_SHIFT;
2419 if (!pfn_valid(pfn))
2420 return -EPERM;
2421
2422 p = pfn_to_page(pfn);
2423 if (p->mapping != adev->mman.bdev.dev_mapping)
2424 return -EPERM;
2425
2426 ptr = kmap(p);
2427 r = copy_from_user(ptr + off, buf, bytes);
2428 kunmap(p);
2429 if (r)
2430 return -EFAULT;
2431
2432 size -= bytes;
2433 *pos += bytes;
2434 result += bytes;
2435 }
2436
2437 return result;
2438}
2439
2440static const struct file_operations amdgpu_ttm_iomem_fops = {
2441 .owner = THIS_MODULE,
2442 .read = amdgpu_iomem_read,
2443 .write = amdgpu_iomem_write,
2444 .llseek = default_llseek
2445};
2446
2447static const struct {
2448 char *name;
2449 const struct file_operations *fops;
2450 int domain;
2451} ttm_debugfs_entries[] = {
2452 { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
2453#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
2454 { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
2455#endif
2456 { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM },
2457};
2458
2459#endif
2460
2461static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
2462{
2463#if defined(CONFIG_DEBUG_FS)
2464 unsigned count;
2465
2466 struct drm_minor *minor = adev->ddev->primary;
2467 struct dentry *ent, *root = minor->debugfs_root;
2468
2469 for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
2470 ent = debugfs_create_file(
2471 ttm_debugfs_entries[count].name,
2472 S_IFREG | S_IRUGO, root,
2473 adev,
2474 ttm_debugfs_entries[count].fops);
2475 if (IS_ERR(ent))
2476 return PTR_ERR(ent);
2477 if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
2478 i_size_write(ent->d_inode, adev->gmc.mc_vram_size);
2479 else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
2480 i_size_write(ent->d_inode, adev->gmc.gart_size);
2481 adev->mman.debugfs_entries[count] = ent;
2482 }
2483
2484 count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);
2485
2486#ifdef CONFIG_SWIOTLB
2487 if (!(adev->need_swiotlb && swiotlb_nr_tbl()))
2488 --count;
2489#endif
2490
2491 return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
2492#else
2493 return 0;
2494#endif
2495}
2496
2497static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
2498{
2499#if defined(CONFIG_DEBUG_FS)
2500 unsigned i;
2501
2502 for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
2503 debugfs_remove(adev->mman.debugfs_entries[i]);
2504#endif
2505}
2506