1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/sched/mm.h>
26#include <linux/dma-fence-array.h>
27#include <drm/drm_gem.h>
28
29#include "display/intel_frontbuffer.h"
30#include "gem/i915_gem_lmem.h"
31#include "gem/i915_gem_tiling.h"
32#include "gt/intel_engine.h"
33#include "gt/intel_engine_heartbeat.h"
34#include "gt/intel_gt.h"
35#include "gt/intel_gt_requests.h"
36
37#include "i915_drv.h"
38#include "i915_gem_evict.h"
39#include "i915_sw_fence_work.h"
40#include "i915_trace.h"
41#include "i915_vma.h"
42#include "i915_vma_resource.h"
43
44static inline void assert_vma_held_evict(const struct i915_vma *vma)
45{
46
47
48
49
50
51 if (kref_read(&vma->vm->ref))
52 assert_object_held_shared(vma->obj);
53}
54
55static struct kmem_cache *slab_vmas;
56
57static struct i915_vma *i915_vma_alloc(void)
58{
59 return kmem_cache_zalloc(slab_vmas, GFP_KERNEL);
60}
61
62static void i915_vma_free(struct i915_vma *vma)
63{
64 return kmem_cache_free(slab_vmas, vma);
65}
66
67#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
68
69#include <linux/stackdepot.h>
70
71static void vma_print_allocator(struct i915_vma *vma, const char *reason)
72{
73 char buf[512];
74
75 if (!vma->node.stack) {
76 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",
77 vma->node.start, vma->node.size, reason);
78 return;
79 }
80
81 stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0);
82 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",
83 vma->node.start, vma->node.size, reason, buf);
84}
85
86#else
87
88static void vma_print_allocator(struct i915_vma *vma, const char *reason)
89{
90}
91
92#endif
93
94static inline struct i915_vma *active_to_vma(struct i915_active *ref)
95{
96 return container_of(ref, typeof(struct i915_vma), active);
97}
98
99static int __i915_vma_active(struct i915_active *ref)
100{
101 return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT;
102}
103
104static void __i915_vma_retire(struct i915_active *ref)
105{
106 i915_vma_put(active_to_vma(ref));
107}
108
109static struct i915_vma *
110vma_create(struct drm_i915_gem_object *obj,
111 struct i915_address_space *vm,
112 const struct i915_ggtt_view *view)
113{
114 struct i915_vma *pos = ERR_PTR(-E2BIG);
115 struct i915_vma *vma;
116 struct rb_node *rb, **p;
117 int err;
118
119
120 GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm);
121
122 vma = i915_vma_alloc();
123 if (vma == NULL)
124 return ERR_PTR(-ENOMEM);
125
126 vma->ops = &vm->vma_ops;
127 vma->obj = obj;
128 vma->size = obj->base.size;
129 vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
130
131 i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0);
132
133
134 if (IS_ENABLED(CONFIG_LOCKDEP)) {
135 fs_reclaim_acquire(GFP_KERNEL);
136 might_lock(&vma->active.mutex);
137 fs_reclaim_release(GFP_KERNEL);
138 }
139
140 INIT_LIST_HEAD(&vma->closed_link);
141 INIT_LIST_HEAD(&vma->obj_link);
142 RB_CLEAR_NODE(&vma->obj_node);
143
144 if (view && view->type != I915_GGTT_VIEW_NORMAL) {
145 vma->ggtt_view = *view;
146 if (view->type == I915_GGTT_VIEW_PARTIAL) {
147 GEM_BUG_ON(range_overflows_t(u64,
148 view->partial.offset,
149 view->partial.size,
150 obj->base.size >> PAGE_SHIFT));
151 vma->size = view->partial.size;
152 vma->size <<= PAGE_SHIFT;
153 GEM_BUG_ON(vma->size > obj->base.size);
154 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
155 vma->size = intel_rotation_info_size(&view->rotated);
156 vma->size <<= PAGE_SHIFT;
157 } else if (view->type == I915_GGTT_VIEW_REMAPPED) {
158 vma->size = intel_remapped_info_size(&view->remapped);
159 vma->size <<= PAGE_SHIFT;
160 }
161 }
162
163 if (unlikely(vma->size > vm->total))
164 goto err_vma;
165
166 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE));
167
168 err = mutex_lock_interruptible(&vm->mutex);
169 if (err) {
170 pos = ERR_PTR(err);
171 goto err_vma;
172 }
173
174 vma->vm = vm;
175 list_add_tail(&vma->vm_link, &vm->unbound_list);
176
177 spin_lock(&obj->vma.lock);
178 if (i915_is_ggtt(vm)) {
179 if (unlikely(overflows_type(vma->size, u32)))
180 goto err_unlock;
181
182 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size,
183 i915_gem_object_get_tiling(obj),
184 i915_gem_object_get_stride(obj));
185 if (unlikely(vma->fence_size < vma->size ||
186 vma->fence_size > vm->total))
187 goto err_unlock;
188
189 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT));
190
191 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size,
192 i915_gem_object_get_tiling(obj),
193 i915_gem_object_get_stride(obj));
194 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
195
196 __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
197 }
198
199 rb = NULL;
200 p = &obj->vma.tree.rb_node;
201 while (*p) {
202 long cmp;
203
204 rb = *p;
205 pos = rb_entry(rb, struct i915_vma, obj_node);
206
207
208
209
210
211
212 cmp = i915_vma_compare(pos, vm, view);
213 if (cmp < 0)
214 p = &rb->rb_right;
215 else if (cmp > 0)
216 p = &rb->rb_left;
217 else
218 goto err_unlock;
219 }
220 rb_link_node(&vma->obj_node, rb, p);
221 rb_insert_color(&vma->obj_node, &obj->vma.tree);
222
223 if (i915_vma_is_ggtt(vma))
224
225
226
227
228
229
230 list_add(&vma->obj_link, &obj->vma.list);
231 else
232 list_add_tail(&vma->obj_link, &obj->vma.list);
233
234 spin_unlock(&obj->vma.lock);
235 mutex_unlock(&vm->mutex);
236
237 return vma;
238
239err_unlock:
240 spin_unlock(&obj->vma.lock);
241 list_del_init(&vma->vm_link);
242 mutex_unlock(&vm->mutex);
243err_vma:
244 i915_vma_free(vma);
245 return pos;
246}
247
248static struct i915_vma *
249i915_vma_lookup(struct drm_i915_gem_object *obj,
250 struct i915_address_space *vm,
251 const struct i915_ggtt_view *view)
252{
253 struct rb_node *rb;
254
255 rb = obj->vma.tree.rb_node;
256 while (rb) {
257 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
258 long cmp;
259
260 cmp = i915_vma_compare(vma, vm, view);
261 if (cmp == 0)
262 return vma;
263
264 if (cmp < 0)
265 rb = rb->rb_right;
266 else
267 rb = rb->rb_left;
268 }
269
270 return NULL;
271}
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286struct i915_vma *
287i915_vma_instance(struct drm_i915_gem_object *obj,
288 struct i915_address_space *vm,
289 const struct i915_ggtt_view *view)
290{
291 struct i915_vma *vma;
292
293 GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));
294 GEM_BUG_ON(!kref_read(&vm->ref));
295
296 spin_lock(&obj->vma.lock);
297 vma = i915_vma_lookup(obj, vm, view);
298 spin_unlock(&obj->vma.lock);
299
300
301 if (unlikely(!vma))
302 vma = vma_create(obj, vm, view);
303
304 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
305 return vma;
306}
307
308struct i915_vma_work {
309 struct dma_fence_work base;
310 struct i915_address_space *vm;
311 struct i915_vm_pt_stash stash;
312 struct i915_vma_resource *vma_res;
313 struct drm_i915_gem_object *pinned;
314 struct i915_sw_dma_fence_cb cb;
315 enum i915_cache_level cache_level;
316 unsigned int flags;
317};
318
319static void __vma_bind(struct dma_fence_work *work)
320{
321 struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
322 struct i915_vma_resource *vma_res = vw->vma_res;
323
324 vma_res->ops->bind_vma(vma_res->vm, &vw->stash,
325 vma_res, vw->cache_level, vw->flags);
326
327}
328
329static void __vma_release(struct dma_fence_work *work)
330{
331 struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
332
333 if (vw->pinned)
334 i915_gem_object_put(vw->pinned);
335
336 i915_vm_free_pt_stash(vw->vm, &vw->stash);
337 if (vw->vma_res)
338 i915_vma_resource_put(vw->vma_res);
339}
340
341static const struct dma_fence_work_ops bind_ops = {
342 .name = "bind",
343 .work = __vma_bind,
344 .release = __vma_release,
345};
346
347struct i915_vma_work *i915_vma_work(void)
348{
349 struct i915_vma_work *vw;
350
351 vw = kzalloc(sizeof(*vw), GFP_KERNEL);
352 if (!vw)
353 return NULL;
354
355 dma_fence_work_init(&vw->base, &bind_ops);
356 vw->base.dma.error = -EAGAIN;
357
358 return vw;
359}
360
361int i915_vma_wait_for_bind(struct i915_vma *vma)
362{
363 int err = 0;
364
365 if (rcu_access_pointer(vma->active.excl.fence)) {
366 struct dma_fence *fence;
367
368 rcu_read_lock();
369 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence);
370 rcu_read_unlock();
371 if (fence) {
372 err = dma_fence_wait(fence, true);
373 dma_fence_put(fence);
374 }
375 }
376
377 return err;
378}
379
380#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
381static int i915_vma_verify_bind_complete(struct i915_vma *vma)
382{
383 struct dma_fence *fence = i915_active_fence_get(&vma->active.excl);
384 int err;
385
386 if (!fence)
387 return 0;
388
389 if (dma_fence_is_signaled(fence))
390 err = fence->error;
391 else
392 err = -EBUSY;
393
394 dma_fence_put(fence);
395
396 return err;
397}
398#else
399#define i915_vma_verify_bind_complete(_vma) 0
400#endif
401
402I915_SELFTEST_EXPORT void
403i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
404 struct i915_vma *vma)
405{
406 struct drm_i915_gem_object *obj = vma->obj;
407
408 i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes,
409 obj->mm.rsgt, i915_gem_object_is_readonly(obj),
410 i915_gem_object_is_lmem(obj), obj->mm.region,
411 vma->ops, vma->private, vma->node.start,
412 vma->node.size, vma->size);
413}
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428int i915_vma_bind(struct i915_vma *vma,
429 enum i915_cache_level cache_level,
430 u32 flags,
431 struct i915_vma_work *work,
432 struct i915_vma_resource *vma_res)
433{
434 u32 bind_flags;
435 u32 vma_flags;
436 int ret;
437
438 lockdep_assert_held(&vma->vm->mutex);
439 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
440 GEM_BUG_ON(vma->size > vma->node.size);
441
442 if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,
443 vma->node.size,
444 vma->vm->total))) {
445 i915_vma_resource_free(vma_res);
446 return -ENODEV;
447 }
448
449 if (GEM_DEBUG_WARN_ON(!flags)) {
450 i915_vma_resource_free(vma_res);
451 return -EINVAL;
452 }
453
454 bind_flags = flags;
455 bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
456
457 vma_flags = atomic_read(&vma->flags);
458 vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
459
460 bind_flags &= ~vma_flags;
461 if (bind_flags == 0) {
462 i915_vma_resource_free(vma_res);
463 return 0;
464 }
465
466 GEM_BUG_ON(!atomic_read(&vma->pages_count));
467
468
469 if (work && bind_flags & vma->vm->bind_async_flags)
470 ret = i915_vma_resource_bind_dep_await(vma->vm,
471 &work->base.chain,
472 vma->node.start,
473 vma->node.size,
474 true,
475 GFP_NOWAIT |
476 __GFP_RETRY_MAYFAIL |
477 __GFP_NOWARN);
478 else
479 ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start,
480 vma->node.size, true);
481 if (ret) {
482 i915_vma_resource_free(vma_res);
483 return ret;
484 }
485
486 if (vma->resource || !vma_res) {
487
488 GEM_WARN_ON(!vma_flags);
489 i915_vma_resource_free(vma_res);
490 } else {
491 i915_vma_resource_init_from_vma(vma_res, vma);
492 vma->resource = vma_res;
493 }
494 trace_i915_vma_bind(vma, bind_flags);
495 if (work && bind_flags & vma->vm->bind_async_flags) {
496 struct dma_fence *prev;
497
498 work->vma_res = i915_vma_resource_get(vma->resource);
499 work->cache_level = cache_level;
500 work->flags = bind_flags;
501
502
503
504
505
506
507
508
509
510
511 prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
512 if (prev) {
513 __i915_sw_fence_await_dma_fence(&work->base.chain,
514 prev,
515 &work->cb);
516 dma_fence_put(prev);
517 }
518
519 work->base.dma.error = 0;
520
521
522
523
524
525
526 if (!work->vma_res->bi.pages_rsgt)
527 work->pinned = i915_gem_object_get(vma->obj);
528 } else {
529 ret = i915_gem_object_wait_moving_fence(vma->obj, true);
530 if (ret) {
531 i915_vma_resource_free(vma->resource);
532 vma->resource = NULL;
533
534 return ret;
535 }
536 vma->ops->bind_vma(vma->vm, NULL, vma->resource, cache_level,
537 bind_flags);
538 }
539
540 set_bit(I915_BO_WAS_BOUND_BIT, &vma->obj->flags);
541
542 atomic_or(bind_flags, &vma->flags);
543 return 0;
544}
545
546void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
547{
548 void __iomem *ptr;
549 int err;
550
551 if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY))
552 return IOMEM_ERR_PTR(-EINVAL);
553
554 if (!i915_gem_object_is_lmem(vma->obj)) {
555 if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
556 err = -ENODEV;
557 goto err;
558 }
559 }
560
561 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
562 GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND));
563 GEM_BUG_ON(i915_vma_verify_bind_complete(vma));
564
565 ptr = READ_ONCE(vma->iomap);
566 if (ptr == NULL) {
567
568
569
570
571
572
573 if (i915_gem_object_is_lmem(vma->obj))
574 ptr = i915_gem_object_lmem_io_map(vma->obj, 0,
575 vma->obj->base.size);
576 else
577 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
578 vma->node.start,
579 vma->node.size);
580 if (ptr == NULL) {
581 err = -ENOMEM;
582 goto err;
583 }
584
585 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) {
586 io_mapping_unmap(ptr);
587 ptr = vma->iomap;
588 }
589 }
590
591 __i915_vma_pin(vma);
592
593 err = i915_vma_pin_fence(vma);
594 if (err)
595 goto err_unpin;
596
597 i915_vma_set_ggtt_write(vma);
598
599
600 return ptr;
601
602err_unpin:
603 __i915_vma_unpin(vma);
604err:
605 return IOMEM_ERR_PTR(err);
606}
607
608void i915_vma_flush_writes(struct i915_vma *vma)
609{
610 if (i915_vma_unset_ggtt_write(vma))
611 intel_gt_flush_ggtt_writes(vma->vm->gt);
612}
613
614void i915_vma_unpin_iomap(struct i915_vma *vma)
615{
616 GEM_BUG_ON(vma->iomap == NULL);
617
618 i915_vma_flush_writes(vma);
619
620 i915_vma_unpin_fence(vma);
621 i915_vma_unpin(vma);
622}
623
624void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
625{
626 struct i915_vma *vma;
627 struct drm_i915_gem_object *obj;
628
629 vma = fetch_and_zero(p_vma);
630 if (!vma)
631 return;
632
633 obj = vma->obj;
634 GEM_BUG_ON(!obj);
635
636 i915_vma_unpin(vma);
637
638 if (flags & I915_VMA_RELEASE_MAP)
639 i915_gem_object_unpin_map(obj);
640
641 i915_gem_object_put(obj);
642}
643
644bool i915_vma_misplaced(const struct i915_vma *vma,
645 u64 size, u64 alignment, u64 flags)
646{
647 if (!drm_mm_node_allocated(&vma->node))
648 return false;
649
650 if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma)))
651 return true;
652
653 if (vma->node.size < size)
654 return true;
655
656 GEM_BUG_ON(alignment && !is_power_of_2(alignment));
657 if (alignment && !IS_ALIGNED(vma->node.start, alignment))
658 return true;
659
660 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
661 return true;
662
663 if (flags & PIN_OFFSET_BIAS &&
664 vma->node.start < (flags & PIN_OFFSET_MASK))
665 return true;
666
667 if (flags & PIN_OFFSET_FIXED &&
668 vma->node.start != (flags & PIN_OFFSET_MASK))
669 return true;
670
671 return false;
672}
673
674void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
675{
676 bool mappable, fenceable;
677
678 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
679 GEM_BUG_ON(!vma->fence_size);
680
681 fenceable = (vma->node.size >= vma->fence_size &&
682 IS_ALIGNED(vma->node.start, vma->fence_alignment));
683
684 mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end;
685
686 if (mappable && fenceable)
687 set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
688 else
689 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
690}
691
692bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
693{
694 struct drm_mm_node *node = &vma->node;
695 struct drm_mm_node *other;
696
697
698
699
700
701
702
703
704 if (!i915_vm_has_cache_coloring(vma->vm))
705 return true;
706
707
708 GEM_BUG_ON(!drm_mm_node_allocated(node));
709 GEM_BUG_ON(list_empty(&node->node_list));
710
711 other = list_prev_entry(node, node_list);
712 if (i915_node_color_differs(other, color) &&
713 !drm_mm_hole_follows(other))
714 return false;
715
716 other = list_next_entry(node, node_list);
717 if (i915_node_color_differs(other, color) &&
718 !drm_mm_hole_follows(node))
719 return false;
720
721 return true;
722}
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738static int
739i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
740 u64 size, u64 alignment, u64 flags)
741{
742 unsigned long color;
743 u64 start, end;
744 int ret;
745
746 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
747 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
748
749 size = max(size, vma->size);
750 alignment = max(alignment, vma->display_alignment);
751 if (flags & PIN_MAPPABLE) {
752 size = max_t(typeof(size), size, vma->fence_size);
753 alignment = max_t(typeof(alignment),
754 alignment, vma->fence_alignment);
755 }
756
757 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
758 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
759 GEM_BUG_ON(!is_power_of_2(alignment));
760
761 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
762 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
763
764 end = vma->vm->total;
765 if (flags & PIN_MAPPABLE)
766 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end);
767 if (flags & PIN_ZONE_4G)
768 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
769 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
770
771 alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj));
772
773
774
775
776 if (NEEDS_COMPACT_PT(vma->vm->i915))
777 size = round_up(size, alignment);
778
779
780
781
782
783 if (size > end) {
784 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",
785 size, flags & PIN_MAPPABLE ? "mappable" : "total",
786 end);
787 return -ENOSPC;
788 }
789
790 color = 0;
791
792 if (i915_vm_has_cache_coloring(vma->vm))
793 color = vma->obj->cache_level;
794
795 if (flags & PIN_OFFSET_FIXED) {
796 u64 offset = flags & PIN_OFFSET_MASK;
797 if (!IS_ALIGNED(offset, alignment) ||
798 range_overflows(offset, size, end))
799 return -EINVAL;
800
801 ret = i915_gem_gtt_reserve(vma->vm, ww, &vma->node,
802 size, offset, color,
803 flags);
804 if (ret)
805 return ret;
806 } else {
807
808
809
810
811
812
813
814
815 if (upper_32_bits(end - 1) &&
816 vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
817
818
819
820
821
822
823 u64 page_alignment =
824 rounddown_pow_of_two(vma->page_sizes.sg |
825 I915_GTT_PAGE_SIZE_2M);
826
827
828
829
830
831
832 GEM_BUG_ON(i915_vma_is_ggtt(vma));
833
834 alignment = max(alignment, page_alignment);
835
836 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
837 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
838 }
839
840 ret = i915_gem_gtt_insert(vma->vm, ww, &vma->node,
841 size, alignment, color,
842 start, end, flags);
843 if (ret)
844 return ret;
845
846 GEM_BUG_ON(vma->node.start < start);
847 GEM_BUG_ON(vma->node.start + vma->node.size > end);
848 }
849 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
850 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
851
852 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
853
854 return 0;
855}
856
857static void
858i915_vma_detach(struct i915_vma *vma)
859{
860 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
861 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
862
863
864
865
866
867
868 list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
869}
870
871static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
872{
873 unsigned int bound;
874
875 bound = atomic_read(&vma->flags);
876
877 if (flags & PIN_VALIDATE) {
878 flags &= I915_VMA_BIND_MASK;
879
880 return (flags & bound) == flags;
881 }
882
883
884 flags &= I915_VMA_BIND_MASK;
885 do {
886 if (unlikely(flags & ~bound))
887 return false;
888
889 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR)))
890 return false;
891
892 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0);
893 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
894
895 return true;
896}
897
898static struct scatterlist *
899rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
900 unsigned int width, unsigned int height,
901 unsigned int src_stride, unsigned int dst_stride,
902 struct sg_table *st, struct scatterlist *sg)
903{
904 unsigned int column, row;
905 unsigned int src_idx;
906
907 for (column = 0; column < width; column++) {
908 unsigned int left;
909
910 src_idx = src_stride * (height - 1) + column + offset;
911 for (row = 0; row < height; row++) {
912 st->nents++;
913
914
915
916
917
918 sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
919 sg_dma_address(sg) =
920 i915_gem_object_get_dma_address(obj, src_idx);
921 sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
922 sg = sg_next(sg);
923 src_idx -= src_stride;
924 }
925
926 left = (dst_stride - height) * I915_GTT_PAGE_SIZE;
927
928 if (!left)
929 continue;
930
931 st->nents++;
932
933
934
935
936
937
938 sg_set_page(sg, NULL, left, 0);
939 sg_dma_address(sg) = 0;
940 sg_dma_len(sg) = left;
941 sg = sg_next(sg);
942 }
943
944 return sg;
945}
946
947static noinline struct sg_table *
948intel_rotate_pages(struct intel_rotation_info *rot_info,
949 struct drm_i915_gem_object *obj)
950{
951 unsigned int size = intel_rotation_info_size(rot_info);
952 struct drm_i915_private *i915 = to_i915(obj->base.dev);
953 struct sg_table *st;
954 struct scatterlist *sg;
955 int ret = -ENOMEM;
956 int i;
957
958
959 st = kmalloc(sizeof(*st), GFP_KERNEL);
960 if (!st)
961 goto err_st_alloc;
962
963 ret = sg_alloc_table(st, size, GFP_KERNEL);
964 if (ret)
965 goto err_sg_alloc;
966
967 st->nents = 0;
968 sg = st->sgl;
969
970 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
971 sg = rotate_pages(obj, rot_info->plane[i].offset,
972 rot_info->plane[i].width, rot_info->plane[i].height,
973 rot_info->plane[i].src_stride,
974 rot_info->plane[i].dst_stride,
975 st, sg);
976
977 return st;
978
979err_sg_alloc:
980 kfree(st);
981err_st_alloc:
982
983 drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
984 obj->base.size, rot_info->plane[0].width,
985 rot_info->plane[0].height, size);
986
987 return ERR_PTR(ret);
988}
989
990static struct scatterlist *
991add_padding_pages(unsigned int count,
992 struct sg_table *st, struct scatterlist *sg)
993{
994 st->nents++;
995
996
997
998
999
1000
1001 sg_set_page(sg, NULL, count * I915_GTT_PAGE_SIZE, 0);
1002 sg_dma_address(sg) = 0;
1003 sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE;
1004 sg = sg_next(sg);
1005
1006 return sg;
1007}
1008
1009static struct scatterlist *
1010remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj,
1011 unsigned int offset, unsigned int alignment_pad,
1012 unsigned int width, unsigned int height,
1013 unsigned int src_stride, unsigned int dst_stride,
1014 struct sg_table *st, struct scatterlist *sg,
1015 unsigned int *gtt_offset)
1016{
1017 unsigned int row;
1018
1019 if (!width || !height)
1020 return sg;
1021
1022 if (alignment_pad)
1023 sg = add_padding_pages(alignment_pad, st, sg);
1024
1025 for (row = 0; row < height; row++) {
1026 unsigned int left = width * I915_GTT_PAGE_SIZE;
1027
1028 while (left) {
1029 dma_addr_t addr;
1030 unsigned int length;
1031
1032
1033
1034
1035
1036
1037
1038 addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1039
1040 length = min(left, length);
1041
1042 st->nents++;
1043
1044 sg_set_page(sg, NULL, length, 0);
1045 sg_dma_address(sg) = addr;
1046 sg_dma_len(sg) = length;
1047 sg = sg_next(sg);
1048
1049 offset += length / I915_GTT_PAGE_SIZE;
1050 left -= length;
1051 }
1052
1053 offset += src_stride - width;
1054
1055 left = (dst_stride - width) * I915_GTT_PAGE_SIZE;
1056
1057 if (!left)
1058 continue;
1059
1060 sg = add_padding_pages(left >> PAGE_SHIFT, st, sg);
1061 }
1062
1063 *gtt_offset += alignment_pad + dst_stride * height;
1064
1065 return sg;
1066}
1067
1068static struct scatterlist *
1069remap_contiguous_pages(struct drm_i915_gem_object *obj,
1070 unsigned int obj_offset,
1071 unsigned int count,
1072 struct sg_table *st, struct scatterlist *sg)
1073{
1074 struct scatterlist *iter;
1075 unsigned int offset;
1076
1077 iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset);
1078 GEM_BUG_ON(!iter);
1079
1080 do {
1081 unsigned int len;
1082
1083 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),
1084 count << PAGE_SHIFT);
1085 sg_set_page(sg, NULL, len, 0);
1086 sg_dma_address(sg) =
1087 sg_dma_address(iter) + (offset << PAGE_SHIFT);
1088 sg_dma_len(sg) = len;
1089
1090 st->nents++;
1091 count -= len >> PAGE_SHIFT;
1092 if (count == 0)
1093 return sg;
1094
1095 sg = __sg_next(sg);
1096 iter = __sg_next(iter);
1097 offset = 0;
1098 } while (1);
1099}
1100
1101static struct scatterlist *
1102remap_linear_color_plane_pages(struct drm_i915_gem_object *obj,
1103 unsigned int obj_offset, unsigned int alignment_pad,
1104 unsigned int size,
1105 struct sg_table *st, struct scatterlist *sg,
1106 unsigned int *gtt_offset)
1107{
1108 if (!size)
1109 return sg;
1110
1111 if (alignment_pad)
1112 sg = add_padding_pages(alignment_pad, st, sg);
1113
1114 sg = remap_contiguous_pages(obj, obj_offset, size, st, sg);
1115 sg = sg_next(sg);
1116
1117 *gtt_offset += alignment_pad + size;
1118
1119 return sg;
1120}
1121
1122static struct scatterlist *
1123remap_color_plane_pages(const struct intel_remapped_info *rem_info,
1124 struct drm_i915_gem_object *obj,
1125 int color_plane,
1126 struct sg_table *st, struct scatterlist *sg,
1127 unsigned int *gtt_offset)
1128{
1129 unsigned int alignment_pad = 0;
1130
1131 if (rem_info->plane_alignment)
1132 alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset;
1133
1134 if (rem_info->plane[color_plane].linear)
1135 sg = remap_linear_color_plane_pages(obj,
1136 rem_info->plane[color_plane].offset,
1137 alignment_pad,
1138 rem_info->plane[color_plane].size,
1139 st, sg,
1140 gtt_offset);
1141
1142 else
1143 sg = remap_tiled_color_plane_pages(obj,
1144 rem_info->plane[color_plane].offset,
1145 alignment_pad,
1146 rem_info->plane[color_plane].width,
1147 rem_info->plane[color_plane].height,
1148 rem_info->plane[color_plane].src_stride,
1149 rem_info->plane[color_plane].dst_stride,
1150 st, sg,
1151 gtt_offset);
1152
1153 return sg;
1154}
1155
1156static noinline struct sg_table *
1157intel_remap_pages(struct intel_remapped_info *rem_info,
1158 struct drm_i915_gem_object *obj)
1159{
1160 unsigned int size = intel_remapped_info_size(rem_info);
1161 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1162 struct sg_table *st;
1163 struct scatterlist *sg;
1164 unsigned int gtt_offset = 0;
1165 int ret = -ENOMEM;
1166 int i;
1167
1168
1169 st = kmalloc(sizeof(*st), GFP_KERNEL);
1170 if (!st)
1171 goto err_st_alloc;
1172
1173 ret = sg_alloc_table(st, size, GFP_KERNEL);
1174 if (ret)
1175 goto err_sg_alloc;
1176
1177 st->nents = 0;
1178 sg = st->sgl;
1179
1180 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++)
1181 sg = remap_color_plane_pages(rem_info, obj, i, st, sg, >t_offset);
1182
1183 i915_sg_trim(st);
1184
1185 return st;
1186
1187err_sg_alloc:
1188 kfree(st);
1189err_st_alloc:
1190
1191 drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1192 obj->base.size, rem_info->plane[0].width,
1193 rem_info->plane[0].height, size);
1194
1195 return ERR_PTR(ret);
1196}
1197
1198static noinline struct sg_table *
1199intel_partial_pages(const struct i915_ggtt_view *view,
1200 struct drm_i915_gem_object *obj)
1201{
1202 struct sg_table *st;
1203 struct scatterlist *sg;
1204 unsigned int count = view->partial.size;
1205 int ret = -ENOMEM;
1206
1207 st = kmalloc(sizeof(*st), GFP_KERNEL);
1208 if (!st)
1209 goto err_st_alloc;
1210
1211 ret = sg_alloc_table(st, count, GFP_KERNEL);
1212 if (ret)
1213 goto err_sg_alloc;
1214
1215 st->nents = 0;
1216
1217 sg = remap_contiguous_pages(obj, view->partial.offset, count, st, st->sgl);
1218
1219 sg_mark_end(sg);
1220 i915_sg_trim(st);
1221
1222 return st;
1223
1224err_sg_alloc:
1225 kfree(st);
1226err_st_alloc:
1227 return ERR_PTR(ret);
1228}
1229
1230static int
1231__i915_vma_get_pages(struct i915_vma *vma)
1232{
1233 struct sg_table *pages;
1234
1235
1236
1237
1238
1239
1240
1241 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1242
1243 switch (vma->ggtt_view.type) {
1244 default:
1245 GEM_BUG_ON(vma->ggtt_view.type);
1246 fallthrough;
1247 case I915_GGTT_VIEW_NORMAL:
1248 pages = vma->obj->mm.pages;
1249 break;
1250
1251 case I915_GGTT_VIEW_ROTATED:
1252 pages =
1253 intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
1254 break;
1255
1256 case I915_GGTT_VIEW_REMAPPED:
1257 pages =
1258 intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
1259 break;
1260
1261 case I915_GGTT_VIEW_PARTIAL:
1262 pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
1263 break;
1264 }
1265
1266 if (IS_ERR(pages)) {
1267 drm_err(&vma->vm->i915->drm,
1268 "Failed to get pages for VMA view type %u (%ld)!\n",
1269 vma->ggtt_view.type, PTR_ERR(pages));
1270 return PTR_ERR(pages);
1271 }
1272
1273 vma->pages = pages;
1274
1275 return 0;
1276}
1277
1278I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
1279{
1280 int err;
1281
1282 if (atomic_add_unless(&vma->pages_count, 1, 0))
1283 return 0;
1284
1285 err = i915_gem_object_pin_pages(vma->obj);
1286 if (err)
1287 return err;
1288
1289 err = __i915_vma_get_pages(vma);
1290 if (err)
1291 goto err_unpin;
1292
1293 vma->page_sizes = vma->obj->mm.page_sizes;
1294 atomic_inc(&vma->pages_count);
1295
1296 return 0;
1297
1298err_unpin:
1299 __i915_gem_object_unpin_pages(vma->obj);
1300
1301 return err;
1302}
1303
1304static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
1305{
1306
1307 GEM_BUG_ON(atomic_read(&vma->pages_count) < count);
1308
1309 if (atomic_sub_return(count, &vma->pages_count) == 0) {
1310 if (vma->pages != vma->obj->mm.pages) {
1311 sg_free_table(vma->pages);
1312 kfree(vma->pages);
1313 }
1314 vma->pages = NULL;
1315
1316 i915_gem_object_unpin_pages(vma->obj);
1317 }
1318}
1319
1320I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma)
1321{
1322 if (atomic_add_unless(&vma->pages_count, -1, 1))
1323 return;
1324
1325 __vma_put_pages(vma, 1);
1326}
1327
1328static void vma_unbind_pages(struct i915_vma *vma)
1329{
1330 unsigned int count;
1331
1332 lockdep_assert_held(&vma->vm->mutex);
1333
1334
1335 count = atomic_read(&vma->pages_count);
1336 count >>= I915_VMA_PAGES_BIAS;
1337 GEM_BUG_ON(!count);
1338
1339 __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS);
1340}
1341
1342int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1343 u64 size, u64 alignment, u64 flags)
1344{
1345 struct i915_vma_work *work = NULL;
1346 struct dma_fence *moving = NULL;
1347 struct i915_vma_resource *vma_res = NULL;
1348 intel_wakeref_t wakeref = 0;
1349 unsigned int bound;
1350 int err;
1351
1352 assert_vma_held(vma);
1353 GEM_BUG_ON(!ww);
1354
1355 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
1356 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
1357
1358 GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)));
1359
1360
1361 if (try_qad_pin(vma, flags))
1362 return 0;
1363
1364 err = i915_vma_get_pages(vma);
1365 if (err)
1366 return err;
1367
1368 if (flags & PIN_GLOBAL)
1369 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
1370
1371 if (flags & vma->vm->bind_async_flags) {
1372
1373 err = i915_vm_lock_objects(vma->vm, ww);
1374 if (err)
1375 goto err_rpm;
1376
1377 work = i915_vma_work();
1378 if (!work) {
1379 err = -ENOMEM;
1380 goto err_rpm;
1381 }
1382
1383 work->vm = vma->vm;
1384
1385 err = i915_gem_object_get_moving_fence(vma->obj, &moving);
1386 if (err)
1387 goto err_rpm;
1388
1389 dma_fence_work_chain(&work->base, moving);
1390
1391
1392 if (vma->vm->allocate_va_range) {
1393 err = i915_vm_alloc_pt_stash(vma->vm,
1394 &work->stash,
1395 vma->size);
1396 if (err)
1397 goto err_fence;
1398
1399 err = i915_vm_map_pt_stash(vma->vm, &work->stash);
1400 if (err)
1401 goto err_fence;
1402 }
1403 }
1404
1405 vma_res = i915_vma_resource_alloc();
1406 if (IS_ERR(vma_res)) {
1407 err = PTR_ERR(vma_res);
1408 goto err_fence;
1409 }
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428 err = mutex_lock_interruptible_nested(&vma->vm->mutex,
1429 !(flags & PIN_GLOBAL));
1430 if (err)
1431 goto err_vma_res;
1432
1433
1434
1435 if (unlikely(i915_vma_is_closed(vma))) {
1436 err = -ENOENT;
1437 goto err_unlock;
1438 }
1439
1440 bound = atomic_read(&vma->flags);
1441 if (unlikely(bound & I915_VMA_ERROR)) {
1442 err = -ENOMEM;
1443 goto err_unlock;
1444 }
1445
1446 if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) {
1447 err = -EAGAIN;
1448 goto err_unlock;
1449 }
1450
1451 if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) {
1452 if (!(flags & PIN_VALIDATE))
1453 __i915_vma_pin(vma);
1454 goto err_unlock;
1455 }
1456
1457 err = i915_active_acquire(&vma->active);
1458 if (err)
1459 goto err_unlock;
1460
1461 if (!(bound & I915_VMA_BIND_MASK)) {
1462 err = i915_vma_insert(vma, ww, size, alignment, flags);
1463 if (err)
1464 goto err_active;
1465
1466 if (i915_is_ggtt(vma->vm))
1467 __i915_vma_set_map_and_fenceable(vma);
1468 }
1469
1470 GEM_BUG_ON(!vma->pages);
1471 err = i915_vma_bind(vma,
1472 vma->obj->cache_level,
1473 flags, work, vma_res);
1474 vma_res = NULL;
1475 if (err)
1476 goto err_remove;
1477
1478
1479 GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound);
1480 atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count);
1481 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1482
1483 if (!(flags & PIN_VALIDATE)) {
1484 __i915_vma_pin(vma);
1485 GEM_BUG_ON(!i915_vma_is_pinned(vma));
1486 }
1487 GEM_BUG_ON(!i915_vma_is_bound(vma, flags));
1488 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
1489
1490err_remove:
1491 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) {
1492 i915_vma_detach(vma);
1493 drm_mm_remove_node(&vma->node);
1494 }
1495err_active:
1496 i915_active_release(&vma->active);
1497err_unlock:
1498 mutex_unlock(&vma->vm->mutex);
1499err_vma_res:
1500 i915_vma_resource_free(vma_res);
1501err_fence:
1502 if (work)
1503 dma_fence_work_commit_imm(&work->base);
1504err_rpm:
1505 if (wakeref)
1506 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
1507
1508 if (moving)
1509 dma_fence_put(moving);
1510
1511 i915_vma_put_pages(vma);
1512 return err;
1513}
1514
1515static void flush_idle_contexts(struct intel_gt *gt)
1516{
1517 struct intel_engine_cs *engine;
1518 enum intel_engine_id id;
1519
1520 for_each_engine(engine, gt, id)
1521 intel_engine_flush_barriers(engine);
1522
1523 intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
1524}
1525
1526static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1527 u32 align, unsigned int flags)
1528{
1529 struct i915_address_space *vm = vma->vm;
1530 int err;
1531
1532 do {
1533 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
1534
1535 if (err != -ENOSPC) {
1536 if (!err) {
1537 err = i915_vma_wait_for_bind(vma);
1538 if (err)
1539 i915_vma_unpin(vma);
1540 }
1541 return err;
1542 }
1543
1544
1545 flush_idle_contexts(vm->gt);
1546 if (mutex_lock_interruptible(&vm->mutex) == 0) {
1547
1548
1549
1550
1551
1552 i915_gem_evict_vm(vm, NULL);
1553 mutex_unlock(&vm->mutex);
1554 }
1555 } while (1);
1556}
1557
1558int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1559 u32 align, unsigned int flags)
1560{
1561 struct i915_gem_ww_ctx _ww;
1562 int err;
1563
1564 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
1565
1566 if (ww)
1567 return __i915_ggtt_pin(vma, ww, align, flags);
1568
1569 lockdep_assert_not_held(&vma->obj->base.resv->lock.base);
1570
1571 for_i915_gem_ww(&_ww, err, true) {
1572 err = i915_gem_object_lock(vma->obj, &_ww);
1573 if (!err)
1574 err = __i915_ggtt_pin(vma, &_ww, align, flags);
1575 }
1576
1577 return err;
1578}
1579
1580static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1581{
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594 GEM_BUG_ON(i915_vma_is_closed(vma));
1595 list_add(&vma->closed_link, >->closed_vma);
1596}
1597
1598void i915_vma_close(struct i915_vma *vma)
1599{
1600 struct intel_gt *gt = vma->vm->gt;
1601 unsigned long flags;
1602
1603 if (i915_vma_is_ggtt(vma))
1604 return;
1605
1606 GEM_BUG_ON(!atomic_read(&vma->open_count));
1607 if (atomic_dec_and_lock_irqsave(&vma->open_count,
1608 >->closed_lock,
1609 flags)) {
1610 __vma_close(vma, gt);
1611 spin_unlock_irqrestore(>->closed_lock, flags);
1612 }
1613}
1614
1615static void __i915_vma_remove_closed(struct i915_vma *vma)
1616{
1617 list_del_init(&vma->closed_link);
1618}
1619
1620void i915_vma_reopen(struct i915_vma *vma)
1621{
1622 struct intel_gt *gt = vma->vm->gt;
1623
1624 spin_lock_irq(>->closed_lock);
1625 if (i915_vma_is_closed(vma))
1626 __i915_vma_remove_closed(vma);
1627 spin_unlock_irq(>->closed_lock);
1628}
1629
1630static void force_unbind(struct i915_vma *vma)
1631{
1632 if (!drm_mm_node_allocated(&vma->node))
1633 return;
1634
1635 atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
1636 WARN_ON(__i915_vma_unbind(vma));
1637 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1638}
1639
1640static void release_references(struct i915_vma *vma, struct intel_gt *gt,
1641 bool vm_ddestroy)
1642{
1643 struct drm_i915_gem_object *obj = vma->obj;
1644
1645 GEM_BUG_ON(i915_vma_is_active(vma));
1646
1647 spin_lock(&obj->vma.lock);
1648 list_del(&vma->obj_link);
1649 if (!RB_EMPTY_NODE(&vma->obj_node))
1650 rb_erase(&vma->obj_node, &obj->vma.tree);
1651
1652 spin_unlock(&obj->vma.lock);
1653
1654 spin_lock_irq(>->closed_lock);
1655 __i915_vma_remove_closed(vma);
1656 spin_unlock_irq(>->closed_lock);
1657
1658 if (vm_ddestroy)
1659 i915_vm_resv_put(vma->vm);
1660
1661 i915_active_fini(&vma->active);
1662 GEM_WARN_ON(vma->resource);
1663 i915_vma_free(vma);
1664}
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692void i915_vma_destroy_locked(struct i915_vma *vma)
1693{
1694 lockdep_assert_held(&vma->vm->mutex);
1695
1696 force_unbind(vma);
1697 list_del_init(&vma->vm_link);
1698 release_references(vma, vma->vm->gt, false);
1699}
1700
1701void i915_vma_destroy(struct i915_vma *vma)
1702{
1703 struct intel_gt *gt;
1704 bool vm_ddestroy;
1705
1706 mutex_lock(&vma->vm->mutex);
1707 force_unbind(vma);
1708 list_del_init(&vma->vm_link);
1709 vm_ddestroy = vma->vm_ddestroy;
1710 vma->vm_ddestroy = false;
1711
1712
1713 gt = vma->vm->gt;
1714 mutex_unlock(&vma->vm->mutex);
1715 release_references(vma, gt, vm_ddestroy);
1716}
1717
1718void i915_vma_parked(struct intel_gt *gt)
1719{
1720 struct i915_vma *vma, *next;
1721 LIST_HEAD(closed);
1722
1723 spin_lock_irq(>->closed_lock);
1724 list_for_each_entry_safe(vma, next, >->closed_vma, closed_link) {
1725 struct drm_i915_gem_object *obj = vma->obj;
1726 struct i915_address_space *vm = vma->vm;
1727
1728
1729
1730 if (!kref_get_unless_zero(&obj->base.refcount))
1731 continue;
1732
1733 if (!i915_vm_tryget(vm)) {
1734 i915_gem_object_put(obj);
1735 continue;
1736 }
1737
1738 list_move(&vma->closed_link, &closed);
1739 }
1740 spin_unlock_irq(>->closed_lock);
1741
1742
1743 list_for_each_entry_safe(vma, next, &closed, closed_link) {
1744 struct drm_i915_gem_object *obj = vma->obj;
1745 struct i915_address_space *vm = vma->vm;
1746
1747 if (i915_gem_object_trylock(obj, NULL)) {
1748 INIT_LIST_HEAD(&vma->closed_link);
1749 i915_vma_destroy(vma);
1750 i915_gem_object_unlock(obj);
1751 } else {
1752
1753 spin_lock_irq(>->closed_lock);
1754 list_add(&vma->closed_link, >->closed_vma);
1755 spin_unlock_irq(>->closed_lock);
1756 }
1757
1758 i915_gem_object_put(obj);
1759 i915_vm_put(vm);
1760 }
1761}
1762
1763static void __i915_vma_iounmap(struct i915_vma *vma)
1764{
1765 GEM_BUG_ON(i915_vma_is_pinned(vma));
1766
1767 if (vma->iomap == NULL)
1768 return;
1769
1770 io_mapping_unmap(vma->iomap);
1771 vma->iomap = NULL;
1772}
1773
1774void i915_vma_revoke_mmap(struct i915_vma *vma)
1775{
1776 struct drm_vma_offset_node *node;
1777 u64 vma_offset;
1778
1779 if (!i915_vma_has_userfault(vma))
1780 return;
1781
1782 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
1783 GEM_BUG_ON(!vma->obj->userfault_count);
1784
1785 node = &vma->mmo->vma_node;
1786 vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
1787 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
1788 drm_vma_node_offset_addr(node) + vma_offset,
1789 vma->size,
1790 1);
1791
1792 i915_vma_unset_userfault(vma);
1793 if (!--vma->obj->userfault_count)
1794 list_del(&vma->obj->userfault_link);
1795}
1796
1797static int
1798__i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1799{
1800 return __i915_request_await_exclusive(rq, &vma->active);
1801}
1802
1803static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1804{
1805 int err;
1806
1807
1808 err = __i915_request_await_bind(rq, vma);
1809 if (err)
1810 return err;
1811
1812 return i915_active_add_request(&vma->active, rq);
1813}
1814
1815int _i915_vma_move_to_active(struct i915_vma *vma,
1816 struct i915_request *rq,
1817 struct dma_fence *fence,
1818 unsigned int flags)
1819{
1820 struct drm_i915_gem_object *obj = vma->obj;
1821 int err;
1822
1823 assert_object_held(obj);
1824
1825 GEM_BUG_ON(!vma->pages);
1826
1827 err = __i915_vma_move_to_active(vma, rq);
1828 if (unlikely(err))
1829 return err;
1830
1831
1832
1833
1834
1835 if (fence && !(flags & __EXEC_OBJECT_NO_RESERVE)) {
1836 struct dma_fence *curr;
1837 int idx;
1838
1839 dma_fence_array_for_each(curr, idx, fence)
1840 ;
1841 err = dma_resv_reserve_fences(vma->obj->base.resv, idx);
1842 if (unlikely(err))
1843 return err;
1844 }
1845
1846 if (flags & EXEC_OBJECT_WRITE) {
1847 struct intel_frontbuffer *front;
1848
1849 front = __intel_frontbuffer_get(obj);
1850 if (unlikely(front)) {
1851 if (intel_frontbuffer_invalidate(front, ORIGIN_CS))
1852 i915_active_add_request(&front->write, rq);
1853 intel_frontbuffer_put(front);
1854 }
1855 }
1856
1857 if (fence) {
1858 struct dma_fence *curr;
1859 enum dma_resv_usage usage;
1860 int idx;
1861
1862 obj->read_domains = 0;
1863 if (flags & EXEC_OBJECT_WRITE) {
1864 usage = DMA_RESV_USAGE_WRITE;
1865 obj->write_domain = I915_GEM_DOMAIN_RENDER;
1866 } else {
1867 usage = DMA_RESV_USAGE_READ;
1868 }
1869
1870 dma_fence_array_for_each(curr, idx, fence)
1871 dma_resv_add_fence(vma->obj->base.resv, curr, usage);
1872 }
1873
1874 if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence)
1875 i915_active_add_request(&vma->fence->active, rq);
1876
1877 obj->read_domains |= I915_GEM_GPU_DOMAINS;
1878 obj->mm.dirty = true;
1879
1880 GEM_BUG_ON(!i915_vma_is_active(vma));
1881 return 0;
1882}
1883
1884struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
1885{
1886 struct i915_vma_resource *vma_res = vma->resource;
1887 struct dma_fence *unbind_fence;
1888
1889 GEM_BUG_ON(i915_vma_is_pinned(vma));
1890 assert_vma_held_evict(vma);
1891
1892 if (i915_vma_is_map_and_fenceable(vma)) {
1893
1894 i915_vma_revoke_mmap(vma);
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909 i915_vma_flush_writes(vma);
1910
1911
1912 i915_vma_revoke_fence(vma);
1913
1914 __i915_vma_iounmap(vma);
1915 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
1916 }
1917 GEM_BUG_ON(vma->fence);
1918 GEM_BUG_ON(i915_vma_has_userfault(vma));
1919
1920
1921 GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt);
1922
1923
1924 vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) &&
1925 kref_read(&vma->vm->ref);
1926 vma_res->skip_pte_rewrite = !kref_read(&vma->vm->ref) ||
1927 vma->vm->skip_pte_rewrite;
1928 trace_i915_vma_unbind(vma);
1929
1930 unbind_fence = i915_vma_resource_unbind(vma_res);
1931 vma->resource = NULL;
1932
1933 atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),
1934 &vma->flags);
1935
1936 i915_vma_detach(vma);
1937
1938 if (!async && unbind_fence) {
1939 dma_fence_wait(unbind_fence, false);
1940 dma_fence_put(unbind_fence);
1941 unbind_fence = NULL;
1942 }
1943
1944
1945
1946
1947
1948
1949
1950 vma_unbind_pages(vma);
1951 return unbind_fence;
1952}
1953
1954int __i915_vma_unbind(struct i915_vma *vma)
1955{
1956 int ret;
1957
1958 lockdep_assert_held(&vma->vm->mutex);
1959 assert_vma_held_evict(vma);
1960
1961 if (!drm_mm_node_allocated(&vma->node))
1962 return 0;
1963
1964 if (i915_vma_is_pinned(vma)) {
1965 vma_print_allocator(vma, "is pinned");
1966 return -EAGAIN;
1967 }
1968
1969
1970
1971
1972
1973
1974 ret = i915_vma_sync(vma);
1975 if (ret)
1976 return ret;
1977
1978 GEM_BUG_ON(i915_vma_is_active(vma));
1979 __i915_vma_evict(vma, false);
1980
1981 drm_mm_remove_node(&vma->node);
1982 return 0;
1983}
1984
1985static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma)
1986{
1987 struct dma_fence *fence;
1988
1989 lockdep_assert_held(&vma->vm->mutex);
1990
1991 if (!drm_mm_node_allocated(&vma->node))
1992 return NULL;
1993
1994 if (i915_vma_is_pinned(vma) ||
1995 &vma->obj->mm.rsgt->table != vma->resource->bi.pages)
1996 return ERR_PTR(-EAGAIN);
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007 if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active,
2008 I915_ACTIVE_AWAIT_EXCL |
2009 I915_ACTIVE_AWAIT_ACTIVE) < 0) {
2010 return ERR_PTR(-EBUSY);
2011 }
2012
2013 fence = __i915_vma_evict(vma, true);
2014
2015 drm_mm_remove_node(&vma->node);
2016
2017 return fence;
2018}
2019
2020int i915_vma_unbind(struct i915_vma *vma)
2021{
2022 struct i915_address_space *vm = vma->vm;
2023 intel_wakeref_t wakeref = 0;
2024 int err;
2025
2026 assert_object_held_shared(vma->obj);
2027
2028
2029 err = i915_vma_sync(vma);
2030 if (err)
2031 return err;
2032
2033 if (!drm_mm_node_allocated(&vma->node))
2034 return 0;
2035
2036 if (i915_vma_is_pinned(vma)) {
2037 vma_print_allocator(vma, "is pinned");
2038 return -EAGAIN;
2039 }
2040
2041 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2042
2043 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
2044
2045 err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref);
2046 if (err)
2047 goto out_rpm;
2048
2049 err = __i915_vma_unbind(vma);
2050 mutex_unlock(&vm->mutex);
2051
2052out_rpm:
2053 if (wakeref)
2054 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
2055 return err;
2056}
2057
2058int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm)
2059{
2060 struct drm_i915_gem_object *obj = vma->obj;
2061 struct i915_address_space *vm = vma->vm;
2062 intel_wakeref_t wakeref = 0;
2063 struct dma_fence *fence;
2064 int err;
2065
2066
2067
2068
2069
2070 assert_object_held(obj);
2071
2072 if (!drm_mm_node_allocated(&vma->node))
2073 return 0;
2074
2075 if (i915_vma_is_pinned(vma)) {
2076 vma_print_allocator(vma, "is pinned");
2077 return -EAGAIN;
2078 }
2079
2080 if (!obj->mm.rsgt)
2081 return -EBUSY;
2082
2083 err = dma_resv_reserve_fences(obj->base.resv, 1);
2084 if (err)
2085 return -EBUSY;
2086
2087
2088
2089
2090
2091
2092 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
2093 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
2094
2095 if (trylock_vm && !mutex_trylock(&vm->mutex)) {
2096 err = -EBUSY;
2097 goto out_rpm;
2098 } else if (!trylock_vm) {
2099 err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref);
2100 if (err)
2101 goto out_rpm;
2102 }
2103
2104 fence = __i915_vma_unbind_async(vma);
2105 mutex_unlock(&vm->mutex);
2106 if (IS_ERR_OR_NULL(fence)) {
2107 err = PTR_ERR_OR_ZERO(fence);
2108 goto out_rpm;
2109 }
2110
2111 dma_resv_add_fence(obj->base.resv, fence, DMA_RESV_USAGE_READ);
2112 dma_fence_put(fence);
2113
2114out_rpm:
2115 if (wakeref)
2116 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
2117 return err;
2118}
2119
2120int i915_vma_unbind_unlocked(struct i915_vma *vma)
2121{
2122 int err;
2123
2124 i915_gem_object_lock(vma->obj, NULL);
2125 err = i915_vma_unbind(vma);
2126 i915_gem_object_unlock(vma->obj);
2127
2128 return err;
2129}
2130
2131struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
2132{
2133 i915_gem_object_make_unshrinkable(vma->obj);
2134 return vma;
2135}
2136
2137void i915_vma_make_shrinkable(struct i915_vma *vma)
2138{
2139 i915_gem_object_make_shrinkable(vma->obj);
2140}
2141
2142void i915_vma_make_purgeable(struct i915_vma *vma)
2143{
2144 i915_gem_object_make_purgeable(vma->obj);
2145}
2146
2147#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2148#include "selftests/i915_vma.c"
2149#endif
2150
2151void i915_vma_module_exit(void)
2152{
2153 kmem_cache_destroy(slab_vmas);
2154}
2155
2156int __init i915_vma_module_init(void)
2157{
2158 slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
2159 if (!slab_vmas)
2160 return -ENOMEM;
2161
2162 return 0;
2163}
2164