1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#include <linux/list.h>
33#include <linux/slab.h>
34#include <drm/drmP.h>
35#include <drm/radeon_drm.h>
36#include <drm/drm_cache.h>
37#include "radeon.h"
38#include "radeon_trace.h"
39
40
41int radeon_ttm_init(struct radeon_device *rdev);
42void radeon_ttm_fini(struct radeon_device *rdev);
43static void radeon_bo_clear_surface_reg(struct radeon_bo *bo);
44
45
46
47
48
49
50static void radeon_update_memory_usage(struct radeon_bo *bo,
51 unsigned mem_type, int sign)
52{
53 struct radeon_device *rdev = bo->rdev;
54 u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT;
55
56 switch (mem_type) {
57 case TTM_PL_TT:
58 if (sign > 0)
59 atomic64_add(size, &rdev->gtt_usage);
60 else
61 atomic64_sub(size, &rdev->gtt_usage);
62 break;
63 case TTM_PL_VRAM:
64 if (sign > 0)
65 atomic64_add(size, &rdev->vram_usage);
66 else
67 atomic64_sub(size, &rdev->vram_usage);
68 break;
69 }
70}
71
72static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
73{
74 struct radeon_bo *bo;
75
76 bo = container_of(tbo, struct radeon_bo, tbo);
77
78 radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1);
79
80 mutex_lock(&bo->rdev->gem.mutex);
81 list_del_init(&bo->list);
82 mutex_unlock(&bo->rdev->gem.mutex);
83 radeon_bo_clear_surface_reg(bo);
84 WARN_ON(!list_empty(&bo->va));
85 drm_gem_object_release(&bo->gem_base);
86 kfree(bo);
87}
88
89bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo)
90{
91 if (bo->destroy == &radeon_ttm_bo_destroy)
92 return true;
93 return false;
94}
95
96void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
97{
98 u32 c = 0, i;
99
100 rbo->placement.placement = rbo->placements;
101 rbo->placement.busy_placement = rbo->placements;
102 if (domain & RADEON_GEM_DOMAIN_VRAM) {
103
104
105
106 if ((rbo->flags & RADEON_GEM_NO_CPU_ACCESS) &&
107 rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size) {
108 rbo->placements[c].fpfn =
109 rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
110 rbo->placements[c++].flags = TTM_PL_FLAG_WC |
111 TTM_PL_FLAG_UNCACHED |
112 TTM_PL_FLAG_VRAM;
113 }
114
115 rbo->placements[c].fpfn = 0;
116 rbo->placements[c++].flags = TTM_PL_FLAG_WC |
117 TTM_PL_FLAG_UNCACHED |
118 TTM_PL_FLAG_VRAM;
119 }
120
121 if (domain & RADEON_GEM_DOMAIN_GTT) {
122 if (rbo->flags & RADEON_GEM_GTT_UC) {
123 rbo->placements[c].fpfn = 0;
124 rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED |
125 TTM_PL_FLAG_TT;
126
127 } else if ((rbo->flags & RADEON_GEM_GTT_WC) ||
128 (rbo->rdev->flags & RADEON_IS_AGP)) {
129 rbo->placements[c].fpfn = 0;
130 rbo->placements[c++].flags = TTM_PL_FLAG_WC |
131 TTM_PL_FLAG_UNCACHED |
132 TTM_PL_FLAG_TT;
133 } else {
134 rbo->placements[c].fpfn = 0;
135 rbo->placements[c++].flags = TTM_PL_FLAG_CACHED |
136 TTM_PL_FLAG_TT;
137 }
138 }
139
140 if (domain & RADEON_GEM_DOMAIN_CPU) {
141 if (rbo->flags & RADEON_GEM_GTT_UC) {
142 rbo->placements[c].fpfn = 0;
143 rbo->placements[c++].flags = TTM_PL_FLAG_UNCACHED |
144 TTM_PL_FLAG_SYSTEM;
145
146 } else if ((rbo->flags & RADEON_GEM_GTT_WC) ||
147 rbo->rdev->flags & RADEON_IS_AGP) {
148 rbo->placements[c].fpfn = 0;
149 rbo->placements[c++].flags = TTM_PL_FLAG_WC |
150 TTM_PL_FLAG_UNCACHED |
151 TTM_PL_FLAG_SYSTEM;
152 } else {
153 rbo->placements[c].fpfn = 0;
154 rbo->placements[c++].flags = TTM_PL_FLAG_CACHED |
155 TTM_PL_FLAG_SYSTEM;
156 }
157 }
158 if (!c) {
159 rbo->placements[c].fpfn = 0;
160 rbo->placements[c++].flags = TTM_PL_MASK_CACHING |
161 TTM_PL_FLAG_SYSTEM;
162 }
163
164 rbo->placement.num_placement = c;
165 rbo->placement.num_busy_placement = c;
166
167 for (i = 0; i < c; ++i) {
168 if ((rbo->flags & RADEON_GEM_CPU_ACCESS) &&
169 (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
170 !rbo->placements[i].fpfn)
171 rbo->placements[i].lpfn =
172 rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
173 else
174 rbo->placements[i].lpfn = 0;
175 }
176}
177
178int radeon_bo_create(struct radeon_device *rdev,
179 unsigned long size, int byte_align, bool kernel,
180 u32 domain, u32 flags, struct sg_table *sg,
181 struct reservation_object *resv,
182 struct radeon_bo **bo_ptr)
183{
184 struct radeon_bo *bo;
185 enum ttm_bo_type type;
186 unsigned long page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
187 size_t acc_size;
188 int r;
189
190 size = ALIGN(size, PAGE_SIZE);
191
192 if (kernel) {
193 type = ttm_bo_type_kernel;
194 } else if (sg) {
195 type = ttm_bo_type_sg;
196 } else {
197 type = ttm_bo_type_device;
198 }
199 *bo_ptr = NULL;
200
201 acc_size = ttm_bo_dma_acc_size(&rdev->mman.bdev, size,
202 sizeof(struct radeon_bo));
203
204 bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);
205 if (bo == NULL)
206 return -ENOMEM;
207 r = drm_gem_object_init(rdev->ddev, &bo->gem_base, size);
208 if (unlikely(r)) {
209 kfree(bo);
210 return r;
211 }
212 bo->rdev = rdev;
213 bo->surface_reg = -1;
214 INIT_LIST_HEAD(&bo->list);
215 INIT_LIST_HEAD(&bo->va);
216 bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM |
217 RADEON_GEM_DOMAIN_GTT |
218 RADEON_GEM_DOMAIN_CPU);
219
220 bo->flags = flags;
221
222 if (!(rdev->flags & RADEON_IS_PCIE))
223 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);
224
225
226
227
228 if (rdev->family >= CHIP_RV610 && rdev->family <= CHIP_RV635)
229 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);
230
231#ifdef CONFIG_X86_32
232
233
234
235 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);
236#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
237
238
239
240
241
242#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
243 thanks to write-combining
244
245 if (bo->flags & RADEON_GEM_GTT_WC)
246 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
247 "better performance thanks to write-combining\n");
248 bo->flags &= ~(RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC);
249#else
250
251
252
253 if (!drm_arch_can_wc_memory())
254 bo->flags &= ~RADEON_GEM_GTT_WC;
255#endif
256
257 radeon_ttm_placement_from_domain(bo, domain);
258
259 down_read(&rdev->pm.mclk_lock);
260 r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,
261 &bo->placement, page_align, !kernel, NULL,
262 acc_size, sg, resv, &radeon_ttm_bo_destroy);
263 up_read(&rdev->pm.mclk_lock);
264 if (unlikely(r != 0)) {
265 return r;
266 }
267 *bo_ptr = bo;
268
269 trace_radeon_bo_create(bo);
270
271 return 0;
272}
273
274int radeon_bo_kmap(struct radeon_bo *bo, void **ptr)
275{
276 bool is_iomem;
277 int r;
278
279 if (bo->kptr) {
280 if (ptr) {
281 *ptr = bo->kptr;
282 }
283 return 0;
284 }
285 r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
286 if (r) {
287 return r;
288 }
289 bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
290 if (ptr) {
291 *ptr = bo->kptr;
292 }
293 radeon_bo_check_tiling(bo, 0, 0);
294 return 0;
295}
296
297void radeon_bo_kunmap(struct radeon_bo *bo)
298{
299 if (bo->kptr == NULL)
300 return;
301 bo->kptr = NULL;
302 radeon_bo_check_tiling(bo, 0, 0);
303 ttm_bo_kunmap(&bo->kmap);
304}
305
306struct radeon_bo *radeon_bo_ref(struct radeon_bo *bo)
307{
308 if (bo == NULL)
309 return NULL;
310
311 ttm_bo_reference(&bo->tbo);
312 return bo;
313}
314
315void radeon_bo_unref(struct radeon_bo **bo)
316{
317 struct ttm_buffer_object *tbo;
318 struct radeon_device *rdev;
319
320 if ((*bo) == NULL)
321 return;
322 rdev = (*bo)->rdev;
323 tbo = &((*bo)->tbo);
324 ttm_bo_unref(&tbo);
325 if (tbo == NULL)
326 *bo = NULL;
327}
328
329int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,
330 u64 *gpu_addr)
331{
332 int r, i;
333
334 if (radeon_ttm_tt_has_userptr(bo->tbo.ttm))
335 return -EPERM;
336
337 if (bo->pin_count) {
338 bo->pin_count++;
339 if (gpu_addr)
340 *gpu_addr = radeon_bo_gpu_offset(bo);
341
342 if (max_offset != 0) {
343 u64 domain_start;
344
345 if (domain == RADEON_GEM_DOMAIN_VRAM)
346 domain_start = bo->rdev->mc.vram_start;
347 else
348 domain_start = bo->rdev->mc.gtt_start;
349 WARN_ON_ONCE(max_offset <
350 (radeon_bo_gpu_offset(bo) - domain_start));
351 }
352
353 return 0;
354 }
355 radeon_ttm_placement_from_domain(bo, domain);
356 for (i = 0; i < bo->placement.num_placement; i++) {
357
358 if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
359 !(bo->flags & RADEON_GEM_NO_CPU_ACCESS) &&
360 (!max_offset || max_offset > bo->rdev->mc.visible_vram_size))
361 bo->placements[i].lpfn =
362 bo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
363 else
364 bo->placements[i].lpfn = max_offset >> PAGE_SHIFT;
365
366 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
367 }
368
369 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
370 if (likely(r == 0)) {
371 bo->pin_count = 1;
372 if (gpu_addr != NULL)
373 *gpu_addr = radeon_bo_gpu_offset(bo);
374 if (domain == RADEON_GEM_DOMAIN_VRAM)
375 bo->rdev->vram_pin_size += radeon_bo_size(bo);
376 else
377 bo->rdev->gart_pin_size += radeon_bo_size(bo);
378 } else {
379 dev_err(bo->rdev->dev, "%p pin failed\n", bo);
380 }
381 return r;
382}
383
384int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr)
385{
386 return radeon_bo_pin_restricted(bo, domain, 0, gpu_addr);
387}
388
389int radeon_bo_unpin(struct radeon_bo *bo)
390{
391 int r, i;
392
393 if (!bo->pin_count) {
394 dev_warn(bo->rdev->dev, "%p unpin not necessary\n", bo);
395 return 0;
396 }
397 bo->pin_count--;
398 if (bo->pin_count)
399 return 0;
400 for (i = 0; i < bo->placement.num_placement; i++) {
401 bo->placements[i].lpfn = 0;
402 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
403 }
404 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
405 if (likely(r == 0)) {
406 if (bo->tbo.mem.mem_type == TTM_PL_VRAM)
407 bo->rdev->vram_pin_size -= radeon_bo_size(bo);
408 else
409 bo->rdev->gart_pin_size -= radeon_bo_size(bo);
410 } else {
411 dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo);
412 }
413 return r;
414}
415
416int radeon_bo_evict_vram(struct radeon_device *rdev)
417{
418
419 if (0 && (rdev->flags & RADEON_IS_IGP)) {
420 if (rdev->mc.igp_sideport_enabled == false)
421
422 return 0;
423 }
424 return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM);
425}
426
427void radeon_bo_force_delete(struct radeon_device *rdev)
428{
429 struct radeon_bo *bo, *n;
430
431 if (list_empty(&rdev->gem.objects)) {
432 return;
433 }
434 dev_err(rdev->dev, "Userspace still has active objects !\n");
435 list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {
436 dev_err(rdev->dev, "%p %p %lu %lu force free\n",
437 &bo->gem_base, bo, (unsigned long)bo->gem_base.size,
438 *((unsigned long *)&bo->gem_base.refcount));
439 mutex_lock(&bo->rdev->gem.mutex);
440 list_del_init(&bo->list);
441 mutex_unlock(&bo->rdev->gem.mutex);
442
443 drm_gem_object_unreference_unlocked(&bo->gem_base);
444 }
445}
446
447int radeon_bo_init(struct radeon_device *rdev)
448{
449
450 if (!rdev->fastfb_working) {
451 rdev->mc.vram_mtrr = arch_phys_wc_add(rdev->mc.aper_base,
452 rdev->mc.aper_size);
453 }
454 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
455 rdev->mc.mc_vram_size >> 20,
456 (unsigned long long)rdev->mc.aper_size >> 20);
457 DRM_INFO("RAM width %dbits %cDR\n",
458 rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S');
459 return radeon_ttm_init(rdev);
460}
461
462void radeon_bo_fini(struct radeon_device *rdev)
463{
464 radeon_ttm_fini(rdev);
465 arch_phys_wc_del(rdev->mc.vram_mtrr);
466}
467
468
469
470static u64 radeon_bo_get_threshold_for_moves(struct radeon_device *rdev)
471{
472 u64 real_vram_size = rdev->mc.real_vram_size;
473 u64 vram_usage = atomic64_read(&rdev->vram_usage);
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514 u64 half_vram = real_vram_size >> 1;
515 u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
516 u64 bytes_moved_threshold = half_free_vram >> 1;
517 return max(bytes_moved_threshold, 1024*1024ull);
518}
519
520int radeon_bo_list_validate(struct radeon_device *rdev,
521 struct ww_acquire_ctx *ticket,
522 struct list_head *head, int ring)
523{
524 struct radeon_bo_list *lobj;
525 struct list_head duplicates;
526 int r;
527 u64 bytes_moved = 0, initial_bytes_moved;
528 u64 bytes_moved_threshold = radeon_bo_get_threshold_for_moves(rdev);
529
530 INIT_LIST_HEAD(&duplicates);
531 r = ttm_eu_reserve_buffers(ticket, head, true, &duplicates);
532 if (unlikely(r != 0)) {
533 return r;
534 }
535
536 list_for_each_entry(lobj, head, tv.head) {
537 struct radeon_bo *bo = lobj->robj;
538 if (!bo->pin_count) {
539 u32 domain = lobj->prefered_domains;
540 u32 allowed = lobj->allowed_domains;
541 u32 current_domain =
542 radeon_mem_type_to_domain(bo->tbo.mem.mem_type);
543
544
545
546
547
548
549
550
551
552 if ((allowed & current_domain) != 0 &&
553 (domain & current_domain) == 0 &&
554 bytes_moved > bytes_moved_threshold) {
555
556 domain = current_domain;
557 }
558
559 retry:
560 radeon_ttm_placement_from_domain(bo, domain);
561 if (ring == R600_RING_TYPE_UVD_INDEX)
562 radeon_uvd_force_into_uvd_segment(bo, allowed);
563
564 initial_bytes_moved = atomic64_read(&rdev->num_bytes_moved);
565 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
566 bytes_moved += atomic64_read(&rdev->num_bytes_moved) -
567 initial_bytes_moved;
568
569 if (unlikely(r)) {
570 if (r != -ERESTARTSYS &&
571 domain != lobj->allowed_domains) {
572 domain = lobj->allowed_domains;
573 goto retry;
574 }
575 ttm_eu_backoff_reservation(ticket, head);
576 return r;
577 }
578 }
579 lobj->gpu_offset = radeon_bo_gpu_offset(bo);
580 lobj->tiling_flags = bo->tiling_flags;
581 }
582
583 list_for_each_entry(lobj, &duplicates, tv.head) {
584 lobj->gpu_offset = radeon_bo_gpu_offset(lobj->robj);
585 lobj->tiling_flags = lobj->robj->tiling_flags;
586 }
587
588 return 0;
589}
590
591int radeon_bo_get_surface_reg(struct radeon_bo *bo)
592{
593 struct radeon_device *rdev = bo->rdev;
594 struct radeon_surface_reg *reg;
595 struct radeon_bo *old_object;
596 int steal;
597 int i;
598
599 lockdep_assert_held(&bo->tbo.resv->lock.base);
600
601 if (!bo->tiling_flags)
602 return 0;
603
604 if (bo->surface_reg >= 0) {
605 reg = &rdev->surface_regs[bo->surface_reg];
606 i = bo->surface_reg;
607 goto out;
608 }
609
610 steal = -1;
611 for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {
612
613 reg = &rdev->surface_regs[i];
614 if (!reg->bo)
615 break;
616
617 old_object = reg->bo;
618 if (old_object->pin_count == 0)
619 steal = i;
620 }
621
622
623 if (i == RADEON_GEM_MAX_SURFACES) {
624 if (steal == -1)
625 return -ENOMEM;
626
627 reg = &rdev->surface_regs[steal];
628 old_object = reg->bo;
629
630 DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object);
631 ttm_bo_unmap_virtual(&old_object->tbo);
632 old_object->surface_reg = -1;
633 i = steal;
634 }
635
636 bo->surface_reg = i;
637 reg->bo = bo;
638
639out:
640 radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch,
641 bo->tbo.mem.start << PAGE_SHIFT,
642 bo->tbo.num_pages << PAGE_SHIFT);
643 return 0;
644}
645
646static void radeon_bo_clear_surface_reg(struct radeon_bo *bo)
647{
648 struct radeon_device *rdev = bo->rdev;
649 struct radeon_surface_reg *reg;
650
651 if (bo->surface_reg == -1)
652 return;
653
654 reg = &rdev->surface_regs[bo->surface_reg];
655 radeon_clear_surface_reg(rdev, bo->surface_reg);
656
657 reg->bo = NULL;
658 bo->surface_reg = -1;
659}
660
661int radeon_bo_set_tiling_flags(struct radeon_bo *bo,
662 uint32_t tiling_flags, uint32_t pitch)
663{
664 struct radeon_device *rdev = bo->rdev;
665 int r;
666
667 if (rdev->family >= CHIP_CEDAR) {
668 unsigned bankw, bankh, mtaspect, tilesplit, stilesplit;
669
670 bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
671 bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
672 mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
673 tilesplit = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
674 stilesplit = (tiling_flags >> RADEON_TILING_EG_STENCIL_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_STENCIL_TILE_SPLIT_MASK;
675 switch (bankw) {
676 case 0:
677 case 1:
678 case 2:
679 case 4:
680 case 8:
681 break;
682 default:
683 return -EINVAL;
684 }
685 switch (bankh) {
686 case 0:
687 case 1:
688 case 2:
689 case 4:
690 case 8:
691 break;
692 default:
693 return -EINVAL;
694 }
695 switch (mtaspect) {
696 case 0:
697 case 1:
698 case 2:
699 case 4:
700 case 8:
701 break;
702 default:
703 return -EINVAL;
704 }
705 if (tilesplit > 6) {
706 return -EINVAL;
707 }
708 if (stilesplit > 6) {
709 return -EINVAL;
710 }
711 }
712 r = radeon_bo_reserve(bo, false);
713 if (unlikely(r != 0))
714 return r;
715 bo->tiling_flags = tiling_flags;
716 bo->pitch = pitch;
717 radeon_bo_unreserve(bo);
718 return 0;
719}
720
721void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
722 uint32_t *tiling_flags,
723 uint32_t *pitch)
724{
725 lockdep_assert_held(&bo->tbo.resv->lock.base);
726
727 if (tiling_flags)
728 *tiling_flags = bo->tiling_flags;
729 if (pitch)
730 *pitch = bo->pitch;
731}
732
733int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
734 bool force_drop)
735{
736 if (!force_drop)
737 lockdep_assert_held(&bo->tbo.resv->lock.base);
738
739 if (!(bo->tiling_flags & RADEON_TILING_SURFACE))
740 return 0;
741
742 if (force_drop) {
743 radeon_bo_clear_surface_reg(bo);
744 return 0;
745 }
746
747 if (bo->tbo.mem.mem_type != TTM_PL_VRAM) {
748 if (!has_moved)
749 return 0;
750
751 if (bo->surface_reg >= 0)
752 radeon_bo_clear_surface_reg(bo);
753 return 0;
754 }
755
756 if ((bo->surface_reg >= 0) && !has_moved)
757 return 0;
758
759 return radeon_bo_get_surface_reg(bo);
760}
761
762void radeon_bo_move_notify(struct ttm_buffer_object *bo,
763 struct ttm_mem_reg *new_mem)
764{
765 struct radeon_bo *rbo;
766
767 if (!radeon_ttm_bo_is_radeon_bo(bo))
768 return;
769
770 rbo = container_of(bo, struct radeon_bo, tbo);
771 radeon_bo_check_tiling(rbo, 0, 1);
772 radeon_vm_bo_invalidate(rbo->rdev, rbo);
773
774
775 if (!new_mem)
776 return;
777
778 radeon_update_memory_usage(rbo, bo->mem.mem_type, -1);
779 radeon_update_memory_usage(rbo, new_mem->mem_type, 1);
780}
781
782int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
783{
784 struct radeon_device *rdev;
785 struct radeon_bo *rbo;
786 unsigned long offset, size, lpfn;
787 int i, r;
788
789 if (!radeon_ttm_bo_is_radeon_bo(bo))
790 return 0;
791 rbo = container_of(bo, struct radeon_bo, tbo);
792 radeon_bo_check_tiling(rbo, 0, 0);
793 rdev = rbo->rdev;
794 if (bo->mem.mem_type != TTM_PL_VRAM)
795 return 0;
796
797 size = bo->mem.num_pages << PAGE_SHIFT;
798 offset = bo->mem.start << PAGE_SHIFT;
799 if ((offset + size) <= rdev->mc.visible_vram_size)
800 return 0;
801
802
803 if (rbo->pin_count > 0)
804 return -EINVAL;
805
806
807 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM);
808 lpfn = rdev->mc.visible_vram_size >> PAGE_SHIFT;
809 for (i = 0; i < rbo->placement.num_placement; i++) {
810
811 if ((rbo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
812 (!rbo->placements[i].lpfn || rbo->placements[i].lpfn > lpfn))
813 rbo->placements[i].lpfn = lpfn;
814 }
815 r = ttm_bo_validate(bo, &rbo->placement, false, false);
816 if (unlikely(r == -ENOMEM)) {
817 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
818 return ttm_bo_validate(bo, &rbo->placement, false, false);
819 } else if (unlikely(r != 0)) {
820 return r;
821 }
822
823 offset = bo->mem.start << PAGE_SHIFT;
824
825 if ((offset + size) > rdev->mc.visible_vram_size)
826 return -EINVAL;
827
828 return 0;
829}
830
831int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait)
832{
833 int r;
834
835 r = ttm_bo_reserve(&bo->tbo, true, no_wait, NULL);
836 if (unlikely(r != 0))
837 return r;
838 if (mem_type)
839 *mem_type = bo->tbo.mem.mem_type;
840
841 r = ttm_bo_wait(&bo->tbo, true, no_wait);
842 ttm_bo_unreserve(&bo->tbo);
843 return r;
844}
845
846
847
848
849
850
851
852
853
854void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence,
855 bool shared)
856{
857 struct reservation_object *resv = bo->tbo.resv;
858
859 if (shared)
860 reservation_object_add_shared_fence(resv, &fence->base);
861 else
862 reservation_object_add_excl_fence(resv, &fence->base);
863}
864