1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <drm/drmP.h>
29#include <drm/drm_vma_manager.h>
30#include <drm/i915_drm.h>
31#include "i915_drv.h"
32#include "i915_gem_clflush.h"
33#include "i915_vgpu.h"
34#include "i915_trace.h"
35#include "intel_drv.h"
36#include "intel_frontbuffer.h"
37#include "intel_mocs.h"
38#include "intel_workarounds.h"
39#include "i915_gemfs.h"
40#include <linux/dma-fence-array.h>
41#include <linux/kthread.h>
42#include <linux/reservation.h>
43#include <linux/shmem_fs.h>
44#include <linux/slab.h>
45#include <linux/stop_machine.h>
46#include <linux/swap.h>
47#include <linux/pci.h>
48#include <linux/dma-buf.h>
49
50static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
51
52static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
53{
54 if (obj->cache_dirty)
55 return false;
56
57 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
58 return true;
59
60 return obj->pin_global;
61}
62
63static int
64insert_mappable_node(struct i915_ggtt *ggtt,
65 struct drm_mm_node *node, u32 size)
66{
67 memset(node, 0, sizeof(*node));
68 return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
69 size, 0, I915_COLOR_UNEVICTABLE,
70 0, ggtt->mappable_end,
71 DRM_MM_INSERT_LOW);
72}
73
74static void
75remove_mappable_node(struct drm_mm_node *node)
76{
77 drm_mm_remove_node(node);
78}
79
80
81static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
82 u64 size)
83{
84 spin_lock(&dev_priv->mm.object_stat_lock);
85 dev_priv->mm.object_count++;
86 dev_priv->mm.object_memory += size;
87 spin_unlock(&dev_priv->mm.object_stat_lock);
88}
89
90static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
91 u64 size)
92{
93 spin_lock(&dev_priv->mm.object_stat_lock);
94 dev_priv->mm.object_count--;
95 dev_priv->mm.object_memory -= size;
96 spin_unlock(&dev_priv->mm.object_stat_lock);
97}
98
99static int
100i915_gem_wait_for_error(struct i915_gpu_error *error)
101{
102 int ret;
103
104 might_sleep();
105
106
107
108
109
110
111 ret = wait_event_interruptible_timeout(error->reset_queue,
112 !i915_reset_backoff(error),
113 I915_RESET_TIMEOUT);
114 if (ret == 0) {
115 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
116 return -EIO;
117 } else if (ret < 0) {
118 return ret;
119 } else {
120 return 0;
121 }
122}
123
124int i915_mutex_lock_interruptible(struct drm_device *dev)
125{
126 struct drm_i915_private *dev_priv = to_i915(dev);
127 int ret;
128
129 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
130 if (ret)
131 return ret;
132
133 ret = mutex_lock_interruptible(&dev->struct_mutex);
134 if (ret)
135 return ret;
136
137 return 0;
138}
139
140static u32 __i915_gem_park(struct drm_i915_private *i915)
141{
142 GEM_TRACE("\n");
143
144 lockdep_assert_held(&i915->drm.struct_mutex);
145 GEM_BUG_ON(i915->gt.active_requests);
146 GEM_BUG_ON(!list_empty(&i915->gt.active_rings));
147
148 if (!i915->gt.awake)
149 return I915_EPOCH_INVALID;
150
151 GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID);
152
153
154
155
156
157
158
159
160
161
162
163
164 synchronize_irq(i915->drm.irq);
165
166 intel_engines_park(i915);
167 i915_timelines_park(i915);
168
169 i915_pmu_gt_parked(i915);
170 i915_vma_parked(i915);
171
172 i915->gt.awake = false;
173
174 if (INTEL_GEN(i915) >= 6)
175 gen6_rps_idle(i915);
176
177 intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ);
178
179 intel_runtime_pm_put(i915);
180
181 return i915->gt.epoch;
182}
183
184void i915_gem_park(struct drm_i915_private *i915)
185{
186 GEM_TRACE("\n");
187
188 lockdep_assert_held(&i915->drm.struct_mutex);
189 GEM_BUG_ON(i915->gt.active_requests);
190
191 if (!i915->gt.awake)
192 return;
193
194
195 mod_delayed_work(i915->wq, &i915->gt.idle_work, msecs_to_jiffies(100));
196}
197
198void i915_gem_unpark(struct drm_i915_private *i915)
199{
200 GEM_TRACE("\n");
201
202 lockdep_assert_held(&i915->drm.struct_mutex);
203 GEM_BUG_ON(!i915->gt.active_requests);
204
205 if (i915->gt.awake)
206 return;
207
208 intel_runtime_pm_get_noresume(i915);
209
210
211
212
213
214
215
216
217
218
219
220
221 intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
222
223 i915->gt.awake = true;
224 if (unlikely(++i915->gt.epoch == 0))
225 i915->gt.epoch = 1;
226
227 intel_enable_gt_powersave(i915);
228 i915_update_gfx_val(i915);
229 if (INTEL_GEN(i915) >= 6)
230 gen6_rps_busy(i915);
231 i915_pmu_gt_unparked(i915);
232
233 intel_engines_unpark(i915);
234
235 i915_queue_hangcheck(i915);
236
237 queue_delayed_work(i915->wq,
238 &i915->gt.retire_work,
239 round_jiffies_up_relative(HZ));
240}
241
242int
243i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
244 struct drm_file *file)
245{
246 struct drm_i915_private *dev_priv = to_i915(dev);
247 struct i915_ggtt *ggtt = &dev_priv->ggtt;
248 struct drm_i915_gem_get_aperture *args = data;
249 struct i915_vma *vma;
250 u64 pinned;
251
252 pinned = ggtt->vm.reserved;
253 mutex_lock(&dev->struct_mutex);
254 list_for_each_entry(vma, &ggtt->vm.active_list, vm_link)
255 if (i915_vma_is_pinned(vma))
256 pinned += vma->node.size;
257 list_for_each_entry(vma, &ggtt->vm.inactive_list, vm_link)
258 if (i915_vma_is_pinned(vma))
259 pinned += vma->node.size;
260 mutex_unlock(&dev->struct_mutex);
261
262 args->aper_size = ggtt->vm.total;
263 args->aper_available_size = args->aper_size - pinned;
264
265 return 0;
266}
267
268static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
269{
270 struct address_space *mapping = obj->base.filp->f_mapping;
271 drm_dma_handle_t *phys;
272 struct sg_table *st;
273 struct scatterlist *sg;
274 char *vaddr;
275 int i;
276 int err;
277
278 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
279 return -EINVAL;
280
281
282
283
284
285 phys = drm_pci_alloc(obj->base.dev,
286 roundup_pow_of_two(obj->base.size),
287 roundup_pow_of_two(obj->base.size));
288 if (!phys)
289 return -ENOMEM;
290
291 vaddr = phys->vaddr;
292 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
293 struct page *page;
294 char *src;
295
296 page = shmem_read_mapping_page(mapping, i);
297 if (IS_ERR(page)) {
298 err = PTR_ERR(page);
299 goto err_phys;
300 }
301
302 src = kmap_atomic(page);
303 memcpy(vaddr, src, PAGE_SIZE);
304 drm_clflush_virt_range(vaddr, PAGE_SIZE);
305 kunmap_atomic(src);
306
307 put_page(page);
308 vaddr += PAGE_SIZE;
309 }
310
311 i915_gem_chipset_flush(to_i915(obj->base.dev));
312
313 st = kmalloc(sizeof(*st), GFP_KERNEL);
314 if (!st) {
315 err = -ENOMEM;
316 goto err_phys;
317 }
318
319 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
320 kfree(st);
321 err = -ENOMEM;
322 goto err_phys;
323 }
324
325 sg = st->sgl;
326 sg->offset = 0;
327 sg->length = obj->base.size;
328
329 sg_dma_address(sg) = phys->busaddr;
330 sg_dma_len(sg) = obj->base.size;
331
332 obj->phys_handle = phys;
333
334 __i915_gem_object_set_pages(obj, st, sg->length);
335
336 return 0;
337
338err_phys:
339 drm_pci_free(obj->base.dev, phys);
340
341 return err;
342}
343
344static void __start_cpu_write(struct drm_i915_gem_object *obj)
345{
346 obj->read_domains = I915_GEM_DOMAIN_CPU;
347 obj->write_domain = I915_GEM_DOMAIN_CPU;
348 if (cpu_write_needs_clflush(obj))
349 obj->cache_dirty = true;
350}
351
352static void
353__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
354 struct sg_table *pages,
355 bool needs_clflush)
356{
357 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
358
359 if (obj->mm.madv == I915_MADV_DONTNEED)
360 obj->mm.dirty = false;
361
362 if (needs_clflush &&
363 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
364 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
365 drm_clflush_sg(pages);
366
367 __start_cpu_write(obj);
368}
369
370static void
371i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
372 struct sg_table *pages)
373{
374 __i915_gem_object_release_shmem(obj, pages, false);
375
376 if (obj->mm.dirty) {
377 struct address_space *mapping = obj->base.filp->f_mapping;
378 char *vaddr = obj->phys_handle->vaddr;
379 int i;
380
381 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
382 struct page *page;
383 char *dst;
384
385 page = shmem_read_mapping_page(mapping, i);
386 if (IS_ERR(page))
387 continue;
388
389 dst = kmap_atomic(page);
390 drm_clflush_virt_range(vaddr, PAGE_SIZE);
391 memcpy(dst, vaddr, PAGE_SIZE);
392 kunmap_atomic(dst);
393
394 set_page_dirty(page);
395 if (obj->mm.madv == I915_MADV_WILLNEED)
396 mark_page_accessed(page);
397 put_page(page);
398 vaddr += PAGE_SIZE;
399 }
400 obj->mm.dirty = false;
401 }
402
403 sg_free_table(pages);
404 kfree(pages);
405
406 drm_pci_free(obj->base.dev, obj->phys_handle);
407}
408
409static void
410i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
411{
412 i915_gem_object_unpin_pages(obj);
413}
414
415static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
416 .get_pages = i915_gem_object_get_pages_phys,
417 .put_pages = i915_gem_object_put_pages_phys,
418 .release = i915_gem_object_release_phys,
419};
420
421static const struct drm_i915_gem_object_ops i915_gem_object_ops;
422
423int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
424{
425 struct i915_vma *vma;
426 LIST_HEAD(still_in_list);
427 int ret;
428
429 lockdep_assert_held(&obj->base.dev->struct_mutex);
430
431
432
433
434
435
436 ret = i915_gem_object_set_to_cpu_domain(obj, false);
437 if (ret)
438 return ret;
439
440 while ((vma = list_first_entry_or_null(&obj->vma_list,
441 struct i915_vma,
442 obj_link))) {
443 list_move_tail(&vma->obj_link, &still_in_list);
444 ret = i915_vma_unbind(vma);
445 if (ret)
446 break;
447 }
448 list_splice(&still_in_list, &obj->vma_list);
449
450 return ret;
451}
452
453static long
454i915_gem_object_wait_fence(struct dma_fence *fence,
455 unsigned int flags,
456 long timeout,
457 struct intel_rps_client *rps_client)
458{
459 struct i915_request *rq;
460
461 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
462
463 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
464 return timeout;
465
466 if (!dma_fence_is_i915(fence))
467 return dma_fence_wait_timeout(fence,
468 flags & I915_WAIT_INTERRUPTIBLE,
469 timeout);
470
471 rq = to_request(fence);
472 if (i915_request_completed(rq))
473 goto out;
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491 if (rps_client && !i915_request_started(rq)) {
492 if (INTEL_GEN(rq->i915) >= 6)
493 gen6_rps_boost(rq, rps_client);
494 }
495
496 timeout = i915_request_wait(rq, flags, timeout);
497
498out:
499 if (flags & I915_WAIT_LOCKED && i915_request_completed(rq))
500 i915_request_retire_upto(rq);
501
502 return timeout;
503}
504
505static long
506i915_gem_object_wait_reservation(struct reservation_object *resv,
507 unsigned int flags,
508 long timeout,
509 struct intel_rps_client *rps_client)
510{
511 unsigned int seq = __read_seqcount_begin(&resv->seq);
512 struct dma_fence *excl;
513 bool prune_fences = false;
514
515 if (flags & I915_WAIT_ALL) {
516 struct dma_fence **shared;
517 unsigned int count, i;
518 int ret;
519
520 ret = reservation_object_get_fences_rcu(resv,
521 &excl, &count, &shared);
522 if (ret)
523 return ret;
524
525 for (i = 0; i < count; i++) {
526 timeout = i915_gem_object_wait_fence(shared[i],
527 flags, timeout,
528 rps_client);
529 if (timeout < 0)
530 break;
531
532 dma_fence_put(shared[i]);
533 }
534
535 for (; i < count; i++)
536 dma_fence_put(shared[i]);
537 kfree(shared);
538
539
540
541
542
543
544
545
546
547
548 prune_fences = count && timeout >= 0;
549 } else {
550 excl = reservation_object_get_excl_rcu(resv);
551 }
552
553 if (excl && timeout >= 0)
554 timeout = i915_gem_object_wait_fence(excl, flags, timeout,
555 rps_client);
556
557 dma_fence_put(excl);
558
559
560
561
562
563
564 if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) {
565 if (reservation_object_trylock(resv)) {
566 if (!__read_seqcount_retry(&resv->seq, seq))
567 reservation_object_add_excl_fence(resv, NULL);
568 reservation_object_unlock(resv);
569 }
570 }
571
572 return timeout;
573}
574
575static void __fence_set_priority(struct dma_fence *fence,
576 const struct i915_sched_attr *attr)
577{
578 struct i915_request *rq;
579 struct intel_engine_cs *engine;
580
581 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
582 return;
583
584 rq = to_request(fence);
585 engine = rq->engine;
586
587 local_bh_disable();
588 rcu_read_lock();
589 if (engine->schedule)
590 engine->schedule(rq, attr);
591 rcu_read_unlock();
592 local_bh_enable();
593}
594
595static void fence_set_priority(struct dma_fence *fence,
596 const struct i915_sched_attr *attr)
597{
598
599 if (dma_fence_is_array(fence)) {
600 struct dma_fence_array *array = to_dma_fence_array(fence);
601 int i;
602
603 for (i = 0; i < array->num_fences; i++)
604 __fence_set_priority(array->fences[i], attr);
605 } else {
606 __fence_set_priority(fence, attr);
607 }
608}
609
610int
611i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
612 unsigned int flags,
613 const struct i915_sched_attr *attr)
614{
615 struct dma_fence *excl;
616
617 if (flags & I915_WAIT_ALL) {
618 struct dma_fence **shared;
619 unsigned int count, i;
620 int ret;
621
622 ret = reservation_object_get_fences_rcu(obj->resv,
623 &excl, &count, &shared);
624 if (ret)
625 return ret;
626
627 for (i = 0; i < count; i++) {
628 fence_set_priority(shared[i], attr);
629 dma_fence_put(shared[i]);
630 }
631
632 kfree(shared);
633 } else {
634 excl = reservation_object_get_excl_rcu(obj->resv);
635 }
636
637 if (excl) {
638 fence_set_priority(excl, attr);
639 dma_fence_put(excl);
640 }
641 return 0;
642}
643
644
645
646
647
648
649
650
651int
652i915_gem_object_wait(struct drm_i915_gem_object *obj,
653 unsigned int flags,
654 long timeout,
655 struct intel_rps_client *rps_client)
656{
657 might_sleep();
658#if IS_ENABLED(CONFIG_LOCKDEP)
659 GEM_BUG_ON(debug_locks &&
660 !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
661 !!(flags & I915_WAIT_LOCKED));
662#endif
663 GEM_BUG_ON(timeout < 0);
664
665 timeout = i915_gem_object_wait_reservation(obj->resv,
666 flags, timeout,
667 rps_client);
668 return timeout < 0 ? timeout : 0;
669}
670
671static struct intel_rps_client *to_rps_client(struct drm_file *file)
672{
673 struct drm_i915_file_private *fpriv = file->driver_priv;
674
675 return &fpriv->rps_client;
676}
677
678static int
679i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
680 struct drm_i915_gem_pwrite *args,
681 struct drm_file *file)
682{
683 void *vaddr = obj->phys_handle->vaddr + args->offset;
684 char __user *user_data = u64_to_user_ptr(args->data_ptr);
685
686
687
688
689 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
690 if (copy_from_user(vaddr, user_data, args->size))
691 return -EFAULT;
692
693 drm_clflush_virt_range(vaddr, args->size);
694 i915_gem_chipset_flush(to_i915(obj->base.dev));
695
696 intel_fb_obj_flush(obj, ORIGIN_CPU);
697 return 0;
698}
699
700void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
701{
702 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
703}
704
705void i915_gem_object_free(struct drm_i915_gem_object *obj)
706{
707 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
708 kmem_cache_free(dev_priv->objects, obj);
709}
710
711static int
712i915_gem_create(struct drm_file *file,
713 struct drm_i915_private *dev_priv,
714 uint64_t size,
715 uint32_t *handle_p)
716{
717 struct drm_i915_gem_object *obj;
718 int ret;
719 u32 handle;
720
721 size = roundup(size, PAGE_SIZE);
722 if (size == 0)
723 return -EINVAL;
724
725
726 obj = i915_gem_object_create(dev_priv, size);
727 if (IS_ERR(obj))
728 return PTR_ERR(obj);
729
730 ret = drm_gem_handle_create(file, &obj->base, &handle);
731
732 i915_gem_object_put(obj);
733 if (ret)
734 return ret;
735
736 *handle_p = handle;
737 return 0;
738}
739
740int
741i915_gem_dumb_create(struct drm_file *file,
742 struct drm_device *dev,
743 struct drm_mode_create_dumb *args)
744{
745
746 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
747 args->size = args->pitch * args->height;
748 return i915_gem_create(file, to_i915(dev),
749 args->size, &args->handle);
750}
751
752static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
753{
754 return !(obj->cache_level == I915_CACHE_NONE ||
755 obj->cache_level == I915_CACHE_WT);
756}
757
758
759
760
761
762
763
764int
765i915_gem_create_ioctl(struct drm_device *dev, void *data,
766 struct drm_file *file)
767{
768 struct drm_i915_private *dev_priv = to_i915(dev);
769 struct drm_i915_gem_create *args = data;
770
771 i915_gem_flush_free_objects(dev_priv);
772
773 return i915_gem_create(file, dev_priv,
774 args->size, &args->handle);
775}
776
777static inline enum fb_op_origin
778fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
779{
780 return (domain == I915_GEM_DOMAIN_GTT ?
781 obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
782}
783
784void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv)
785{
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805 wmb();
806
807 if (INTEL_INFO(dev_priv)->has_coherent_ggtt)
808 return;
809
810 i915_gem_chipset_flush(dev_priv);
811
812 intel_runtime_pm_get(dev_priv);
813 spin_lock_irq(&dev_priv->uncore.lock);
814
815 POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE));
816
817 spin_unlock_irq(&dev_priv->uncore.lock);
818 intel_runtime_pm_put(dev_priv);
819}
820
821static void
822flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
823{
824 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
825 struct i915_vma *vma;
826
827 if (!(obj->write_domain & flush_domains))
828 return;
829
830 switch (obj->write_domain) {
831 case I915_GEM_DOMAIN_GTT:
832 i915_gem_flush_ggtt_writes(dev_priv);
833
834 intel_fb_obj_flush(obj,
835 fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
836
837 for_each_ggtt_vma(vma, obj) {
838 if (vma->iomap)
839 continue;
840
841 i915_vma_unset_ggtt_write(vma);
842 }
843 break;
844
845 case I915_GEM_DOMAIN_WC:
846 wmb();
847 break;
848
849 case I915_GEM_DOMAIN_CPU:
850 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
851 break;
852
853 case I915_GEM_DOMAIN_RENDER:
854 if (gpu_write_needs_clflush(obj))
855 obj->cache_dirty = true;
856 break;
857 }
858
859 obj->write_domain = 0;
860}
861
862static inline int
863__copy_to_user_swizzled(char __user *cpu_vaddr,
864 const char *gpu_vaddr, int gpu_offset,
865 int length)
866{
867 int ret, cpu_offset = 0;
868
869 while (length > 0) {
870 int cacheline_end = ALIGN(gpu_offset + 1, 64);
871 int this_length = min(cacheline_end - gpu_offset, length);
872 int swizzled_gpu_offset = gpu_offset ^ 64;
873
874 ret = __copy_to_user(cpu_vaddr + cpu_offset,
875 gpu_vaddr + swizzled_gpu_offset,
876 this_length);
877 if (ret)
878 return ret + length;
879
880 cpu_offset += this_length;
881 gpu_offset += this_length;
882 length -= this_length;
883 }
884
885 return 0;
886}
887
888static inline int
889__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
890 const char __user *cpu_vaddr,
891 int length)
892{
893 int ret, cpu_offset = 0;
894
895 while (length > 0) {
896 int cacheline_end = ALIGN(gpu_offset + 1, 64);
897 int this_length = min(cacheline_end - gpu_offset, length);
898 int swizzled_gpu_offset = gpu_offset ^ 64;
899
900 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
901 cpu_vaddr + cpu_offset,
902 this_length);
903 if (ret)
904 return ret + length;
905
906 cpu_offset += this_length;
907 gpu_offset += this_length;
908 length -= this_length;
909 }
910
911 return 0;
912}
913
914
915
916
917
918
919int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
920 unsigned int *needs_clflush)
921{
922 int ret;
923
924 lockdep_assert_held(&obj->base.dev->struct_mutex);
925
926 *needs_clflush = 0;
927 if (!i915_gem_object_has_struct_page(obj))
928 return -ENODEV;
929
930 ret = i915_gem_object_wait(obj,
931 I915_WAIT_INTERRUPTIBLE |
932 I915_WAIT_LOCKED,
933 MAX_SCHEDULE_TIMEOUT,
934 NULL);
935 if (ret)
936 return ret;
937
938 ret = i915_gem_object_pin_pages(obj);
939 if (ret)
940 return ret;
941
942 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
943 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
944 ret = i915_gem_object_set_to_cpu_domain(obj, false);
945 if (ret)
946 goto err_unpin;
947 else
948 goto out;
949 }
950
951 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
952
953
954
955
956
957
958 if (!obj->cache_dirty &&
959 !(obj->read_domains & I915_GEM_DOMAIN_CPU))
960 *needs_clflush = CLFLUSH_BEFORE;
961
962out:
963
964 return 0;
965
966err_unpin:
967 i915_gem_object_unpin_pages(obj);
968 return ret;
969}
970
971int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
972 unsigned int *needs_clflush)
973{
974 int ret;
975
976 lockdep_assert_held(&obj->base.dev->struct_mutex);
977
978 *needs_clflush = 0;
979 if (!i915_gem_object_has_struct_page(obj))
980 return -ENODEV;
981
982 ret = i915_gem_object_wait(obj,
983 I915_WAIT_INTERRUPTIBLE |
984 I915_WAIT_LOCKED |
985 I915_WAIT_ALL,
986 MAX_SCHEDULE_TIMEOUT,
987 NULL);
988 if (ret)
989 return ret;
990
991 ret = i915_gem_object_pin_pages(obj);
992 if (ret)
993 return ret;
994
995 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
996 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
997 ret = i915_gem_object_set_to_cpu_domain(obj, true);
998 if (ret)
999 goto err_unpin;
1000 else
1001 goto out;
1002 }
1003
1004 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
1005
1006
1007
1008
1009
1010
1011 if (!obj->cache_dirty) {
1012 *needs_clflush |= CLFLUSH_AFTER;
1013
1014
1015
1016
1017
1018 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
1019 *needs_clflush |= CLFLUSH_BEFORE;
1020 }
1021
1022out:
1023 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1024 obj->mm.dirty = true;
1025
1026 return 0;
1027
1028err_unpin:
1029 i915_gem_object_unpin_pages(obj);
1030 return ret;
1031}
1032
1033static void
1034shmem_clflush_swizzled_range(char *addr, unsigned long length,
1035 bool swizzled)
1036{
1037 if (unlikely(swizzled)) {
1038 unsigned long start = (unsigned long) addr;
1039 unsigned long end = (unsigned long) addr + length;
1040
1041
1042
1043
1044
1045 start = round_down(start, 128);
1046 end = round_up(end, 128);
1047
1048 drm_clflush_virt_range((void *)start, end - start);
1049 } else {
1050 drm_clflush_virt_range(addr, length);
1051 }
1052
1053}
1054
1055
1056
1057static int
1058shmem_pread_slow(struct page *page, int offset, int length,
1059 char __user *user_data,
1060 bool page_do_bit17_swizzling, bool needs_clflush)
1061{
1062 char *vaddr;
1063 int ret;
1064
1065 vaddr = kmap(page);
1066 if (needs_clflush)
1067 shmem_clflush_swizzled_range(vaddr + offset, length,
1068 page_do_bit17_swizzling);
1069
1070 if (page_do_bit17_swizzling)
1071 ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
1072 else
1073 ret = __copy_to_user(user_data, vaddr + offset, length);
1074 kunmap(page);
1075
1076 return ret ? - EFAULT : 0;
1077}
1078
1079static int
1080shmem_pread(struct page *page, int offset, int length, char __user *user_data,
1081 bool page_do_bit17_swizzling, bool needs_clflush)
1082{
1083 int ret;
1084
1085 ret = -ENODEV;
1086 if (!page_do_bit17_swizzling) {
1087 char *vaddr = kmap_atomic(page);
1088
1089 if (needs_clflush)
1090 drm_clflush_virt_range(vaddr + offset, length);
1091 ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
1092 kunmap_atomic(vaddr);
1093 }
1094 if (ret == 0)
1095 return 0;
1096
1097 return shmem_pread_slow(page, offset, length, user_data,
1098 page_do_bit17_swizzling, needs_clflush);
1099}
1100
1101static int
1102i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
1103 struct drm_i915_gem_pread *args)
1104{
1105 char __user *user_data;
1106 u64 remain;
1107 unsigned int obj_do_bit17_swizzling;
1108 unsigned int needs_clflush;
1109 unsigned int idx, offset;
1110 int ret;
1111
1112 obj_do_bit17_swizzling = 0;
1113 if (i915_gem_object_needs_bit17_swizzle(obj))
1114 obj_do_bit17_swizzling = BIT(17);
1115
1116 ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
1117 if (ret)
1118 return ret;
1119
1120 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
1121 mutex_unlock(&obj->base.dev->struct_mutex);
1122 if (ret)
1123 return ret;
1124
1125 remain = args->size;
1126 user_data = u64_to_user_ptr(args->data_ptr);
1127 offset = offset_in_page(args->offset);
1128 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
1129 struct page *page = i915_gem_object_get_page(obj, idx);
1130 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
1131
1132 ret = shmem_pread(page, offset, length, user_data,
1133 page_to_phys(page) & obj_do_bit17_swizzling,
1134 needs_clflush);
1135 if (ret)
1136 break;
1137
1138 remain -= length;
1139 user_data += length;
1140 offset = 0;
1141 }
1142
1143 i915_gem_obj_finish_shmem_access(obj);
1144 return ret;
1145}
1146
1147static inline bool
1148gtt_user_read(struct io_mapping *mapping,
1149 loff_t base, int offset,
1150 char __user *user_data, int length)
1151{
1152 void __iomem *vaddr;
1153 unsigned long unwritten;
1154
1155
1156 vaddr = io_mapping_map_atomic_wc(mapping, base);
1157 unwritten = __copy_to_user_inatomic(user_data,
1158 (void __force *)vaddr + offset,
1159 length);
1160 io_mapping_unmap_atomic(vaddr);
1161 if (unwritten) {
1162 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
1163 unwritten = copy_to_user(user_data,
1164 (void __force *)vaddr + offset,
1165 length);
1166 io_mapping_unmap(vaddr);
1167 }
1168 return unwritten;
1169}
1170
1171static int
1172i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
1173 const struct drm_i915_gem_pread *args)
1174{
1175 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1176 struct i915_ggtt *ggtt = &i915->ggtt;
1177 struct drm_mm_node node;
1178 struct i915_vma *vma;
1179 void __user *user_data;
1180 u64 remain, offset;
1181 int ret;
1182
1183 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1184 if (ret)
1185 return ret;
1186
1187 intel_runtime_pm_get(i915);
1188 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1189 PIN_MAPPABLE |
1190 PIN_NONFAULT |
1191 PIN_NONBLOCK);
1192 if (!IS_ERR(vma)) {
1193 node.start = i915_ggtt_offset(vma);
1194 node.allocated = false;
1195 ret = i915_vma_put_fence(vma);
1196 if (ret) {
1197 i915_vma_unpin(vma);
1198 vma = ERR_PTR(ret);
1199 }
1200 }
1201 if (IS_ERR(vma)) {
1202 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1203 if (ret)
1204 goto out_unlock;
1205 GEM_BUG_ON(!node.allocated);
1206 }
1207
1208 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1209 if (ret)
1210 goto out_unpin;
1211
1212 mutex_unlock(&i915->drm.struct_mutex);
1213
1214 user_data = u64_to_user_ptr(args->data_ptr);
1215 remain = args->size;
1216 offset = args->offset;
1217
1218 while (remain > 0) {
1219
1220
1221
1222
1223
1224
1225 u32 page_base = node.start;
1226 unsigned page_offset = offset_in_page(offset);
1227 unsigned page_length = PAGE_SIZE - page_offset;
1228 page_length = remain < page_length ? remain : page_length;
1229 if (node.allocated) {
1230 wmb();
1231 ggtt->vm.insert_page(&ggtt->vm,
1232 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1233 node.start, I915_CACHE_NONE, 0);
1234 wmb();
1235 } else {
1236 page_base += offset & PAGE_MASK;
1237 }
1238
1239 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
1240 user_data, page_length)) {
1241 ret = -EFAULT;
1242 break;
1243 }
1244
1245 remain -= page_length;
1246 user_data += page_length;
1247 offset += page_length;
1248 }
1249
1250 mutex_lock(&i915->drm.struct_mutex);
1251out_unpin:
1252 if (node.allocated) {
1253 wmb();
1254 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
1255 remove_mappable_node(&node);
1256 } else {
1257 i915_vma_unpin(vma);
1258 }
1259out_unlock:
1260 intel_runtime_pm_put(i915);
1261 mutex_unlock(&i915->drm.struct_mutex);
1262
1263 return ret;
1264}
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274int
1275i915_gem_pread_ioctl(struct drm_device *dev, void *data,
1276 struct drm_file *file)
1277{
1278 struct drm_i915_gem_pread *args = data;
1279 struct drm_i915_gem_object *obj;
1280 int ret;
1281
1282 if (args->size == 0)
1283 return 0;
1284
1285 if (!access_ok(VERIFY_WRITE,
1286 u64_to_user_ptr(args->data_ptr),
1287 args->size))
1288 return -EFAULT;
1289
1290 obj = i915_gem_object_lookup(file, args->handle);
1291 if (!obj)
1292 return -ENOENT;
1293
1294
1295 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
1296 ret = -EINVAL;
1297 goto out;
1298 }
1299
1300 trace_i915_gem_object_pread(obj, args->offset, args->size);
1301
1302 ret = i915_gem_object_wait(obj,
1303 I915_WAIT_INTERRUPTIBLE,
1304 MAX_SCHEDULE_TIMEOUT,
1305 to_rps_client(file));
1306 if (ret)
1307 goto out;
1308
1309 ret = i915_gem_object_pin_pages(obj);
1310 if (ret)
1311 goto out;
1312
1313 ret = i915_gem_shmem_pread(obj, args);
1314 if (ret == -EFAULT || ret == -ENODEV)
1315 ret = i915_gem_gtt_pread(obj, args);
1316
1317 i915_gem_object_unpin_pages(obj);
1318out:
1319 i915_gem_object_put(obj);
1320 return ret;
1321}
1322
1323
1324
1325
1326
1327static inline bool
1328ggtt_write(struct io_mapping *mapping,
1329 loff_t base, int offset,
1330 char __user *user_data, int length)
1331{
1332 void __iomem *vaddr;
1333 unsigned long unwritten;
1334
1335
1336 vaddr = io_mapping_map_atomic_wc(mapping, base);
1337 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
1338 user_data, length);
1339 io_mapping_unmap_atomic(vaddr);
1340 if (unwritten) {
1341 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
1342 unwritten = copy_from_user((void __force *)vaddr + offset,
1343 user_data, length);
1344 io_mapping_unmap(vaddr);
1345 }
1346
1347 return unwritten;
1348}
1349
1350
1351
1352
1353
1354
1355
1356static int
1357i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
1358 const struct drm_i915_gem_pwrite *args)
1359{
1360 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1361 struct i915_ggtt *ggtt = &i915->ggtt;
1362 struct drm_mm_node node;
1363 struct i915_vma *vma;
1364 u64 remain, offset;
1365 void __user *user_data;
1366 int ret;
1367
1368 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1369 if (ret)
1370 return ret;
1371
1372 if (i915_gem_object_has_struct_page(obj)) {
1373
1374
1375
1376
1377
1378
1379
1380 if (!intel_runtime_pm_get_if_in_use(i915)) {
1381 ret = -EFAULT;
1382 goto out_unlock;
1383 }
1384 } else {
1385
1386 intel_runtime_pm_get(i915);
1387 }
1388
1389 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1390 PIN_MAPPABLE |
1391 PIN_NONFAULT |
1392 PIN_NONBLOCK);
1393 if (!IS_ERR(vma)) {
1394 node.start = i915_ggtt_offset(vma);
1395 node.allocated = false;
1396 ret = i915_vma_put_fence(vma);
1397 if (ret) {
1398 i915_vma_unpin(vma);
1399 vma = ERR_PTR(ret);
1400 }
1401 }
1402 if (IS_ERR(vma)) {
1403 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1404 if (ret)
1405 goto out_rpm;
1406 GEM_BUG_ON(!node.allocated);
1407 }
1408
1409 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1410 if (ret)
1411 goto out_unpin;
1412
1413 mutex_unlock(&i915->drm.struct_mutex);
1414
1415 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1416
1417 user_data = u64_to_user_ptr(args->data_ptr);
1418 offset = args->offset;
1419 remain = args->size;
1420 while (remain) {
1421
1422
1423
1424
1425
1426
1427 u32 page_base = node.start;
1428 unsigned int page_offset = offset_in_page(offset);
1429 unsigned int page_length = PAGE_SIZE - page_offset;
1430 page_length = remain < page_length ? remain : page_length;
1431 if (node.allocated) {
1432 wmb();
1433 ggtt->vm.insert_page(&ggtt->vm,
1434 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1435 node.start, I915_CACHE_NONE, 0);
1436 wmb();
1437 } else {
1438 page_base += offset & PAGE_MASK;
1439 }
1440
1441
1442
1443
1444
1445
1446 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
1447 user_data, page_length)) {
1448 ret = -EFAULT;
1449 break;
1450 }
1451
1452 remain -= page_length;
1453 user_data += page_length;
1454 offset += page_length;
1455 }
1456 intel_fb_obj_flush(obj, ORIGIN_CPU);
1457
1458 mutex_lock(&i915->drm.struct_mutex);
1459out_unpin:
1460 if (node.allocated) {
1461 wmb();
1462 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
1463 remove_mappable_node(&node);
1464 } else {
1465 i915_vma_unpin(vma);
1466 }
1467out_rpm:
1468 intel_runtime_pm_put(i915);
1469out_unlock:
1470 mutex_unlock(&i915->drm.struct_mutex);
1471 return ret;
1472}
1473
1474static int
1475shmem_pwrite_slow(struct page *page, int offset, int length,
1476 char __user *user_data,
1477 bool page_do_bit17_swizzling,
1478 bool needs_clflush_before,
1479 bool needs_clflush_after)
1480{
1481 char *vaddr;
1482 int ret;
1483
1484 vaddr = kmap(page);
1485 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1486 shmem_clflush_swizzled_range(vaddr + offset, length,
1487 page_do_bit17_swizzling);
1488 if (page_do_bit17_swizzling)
1489 ret = __copy_from_user_swizzled(vaddr, offset, user_data,
1490 length);
1491 else
1492 ret = __copy_from_user(vaddr + offset, user_data, length);
1493 if (needs_clflush_after)
1494 shmem_clflush_swizzled_range(vaddr + offset, length,
1495 page_do_bit17_swizzling);
1496 kunmap(page);
1497
1498 return ret ? -EFAULT : 0;
1499}
1500
1501
1502
1503
1504
1505
1506static int
1507shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
1508 bool page_do_bit17_swizzling,
1509 bool needs_clflush_before,
1510 bool needs_clflush_after)
1511{
1512 int ret;
1513
1514 ret = -ENODEV;
1515 if (!page_do_bit17_swizzling) {
1516 char *vaddr = kmap_atomic(page);
1517
1518 if (needs_clflush_before)
1519 drm_clflush_virt_range(vaddr + offset, len);
1520 ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
1521 if (needs_clflush_after)
1522 drm_clflush_virt_range(vaddr + offset, len);
1523
1524 kunmap_atomic(vaddr);
1525 }
1526 if (ret == 0)
1527 return ret;
1528
1529 return shmem_pwrite_slow(page, offset, len, user_data,
1530 page_do_bit17_swizzling,
1531 needs_clflush_before,
1532 needs_clflush_after);
1533}
1534
1535static int
1536i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
1537 const struct drm_i915_gem_pwrite *args)
1538{
1539 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1540 void __user *user_data;
1541 u64 remain;
1542 unsigned int obj_do_bit17_swizzling;
1543 unsigned int partial_cacheline_write;
1544 unsigned int needs_clflush;
1545 unsigned int offset, idx;
1546 int ret;
1547
1548 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1549 if (ret)
1550 return ret;
1551
1552 ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
1553 mutex_unlock(&i915->drm.struct_mutex);
1554 if (ret)
1555 return ret;
1556
1557 obj_do_bit17_swizzling = 0;
1558 if (i915_gem_object_needs_bit17_swizzle(obj))
1559 obj_do_bit17_swizzling = BIT(17);
1560
1561
1562
1563
1564
1565 partial_cacheline_write = 0;
1566 if (needs_clflush & CLFLUSH_BEFORE)
1567 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
1568
1569 user_data = u64_to_user_ptr(args->data_ptr);
1570 remain = args->size;
1571 offset = offset_in_page(args->offset);
1572 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
1573 struct page *page = i915_gem_object_get_page(obj, idx);
1574 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
1575
1576 ret = shmem_pwrite(page, offset, length, user_data,
1577 page_to_phys(page) & obj_do_bit17_swizzling,
1578 (offset | length) & partial_cacheline_write,
1579 needs_clflush & CLFLUSH_AFTER);
1580 if (ret)
1581 break;
1582
1583 remain -= length;
1584 user_data += length;
1585 offset = 0;
1586 }
1587
1588 intel_fb_obj_flush(obj, ORIGIN_CPU);
1589 i915_gem_obj_finish_shmem_access(obj);
1590 return ret;
1591}
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601int
1602i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1603 struct drm_file *file)
1604{
1605 struct drm_i915_gem_pwrite *args = data;
1606 struct drm_i915_gem_object *obj;
1607 int ret;
1608
1609 if (args->size == 0)
1610 return 0;
1611
1612 if (!access_ok(VERIFY_READ,
1613 u64_to_user_ptr(args->data_ptr),
1614 args->size))
1615 return -EFAULT;
1616
1617 obj = i915_gem_object_lookup(file, args->handle);
1618 if (!obj)
1619 return -ENOENT;
1620
1621
1622 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
1623 ret = -EINVAL;
1624 goto err;
1625 }
1626
1627
1628 if (i915_gem_object_is_readonly(obj)) {
1629 ret = -EINVAL;
1630 goto err;
1631 }
1632
1633 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1634
1635 ret = -ENODEV;
1636 if (obj->ops->pwrite)
1637 ret = obj->ops->pwrite(obj, args);
1638 if (ret != -ENODEV)
1639 goto err;
1640
1641 ret = i915_gem_object_wait(obj,
1642 I915_WAIT_INTERRUPTIBLE |
1643 I915_WAIT_ALL,
1644 MAX_SCHEDULE_TIMEOUT,
1645 to_rps_client(file));
1646 if (ret)
1647 goto err;
1648
1649 ret = i915_gem_object_pin_pages(obj);
1650 if (ret)
1651 goto err;
1652
1653 ret = -EFAULT;
1654
1655
1656
1657
1658
1659
1660 if (!i915_gem_object_has_struct_page(obj) ||
1661 cpu_write_needs_clflush(obj))
1662
1663
1664
1665
1666 ret = i915_gem_gtt_pwrite_fast(obj, args);
1667
1668 if (ret == -EFAULT || ret == -ENOSPC) {
1669 if (obj->phys_handle)
1670 ret = i915_gem_phys_pwrite(obj, args, file);
1671 else
1672 ret = i915_gem_shmem_pwrite(obj, args);
1673 }
1674
1675 i915_gem_object_unpin_pages(obj);
1676err:
1677 i915_gem_object_put(obj);
1678 return ret;
1679}
1680
1681static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
1682{
1683 struct drm_i915_private *i915;
1684 struct list_head *list;
1685 struct i915_vma *vma;
1686
1687 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1688
1689 for_each_ggtt_vma(vma, obj) {
1690 if (i915_vma_is_active(vma))
1691 continue;
1692
1693 if (!drm_mm_node_allocated(&vma->node))
1694 continue;
1695
1696 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1697 }
1698
1699 i915 = to_i915(obj->base.dev);
1700 spin_lock(&i915->mm.obj_lock);
1701 list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
1702 list_move_tail(&obj->mm.link, list);
1703 spin_unlock(&i915->mm.obj_lock);
1704}
1705
1706
1707
1708
1709
1710
1711
1712
1713int
1714i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1715 struct drm_file *file)
1716{
1717 struct drm_i915_gem_set_domain *args = data;
1718 struct drm_i915_gem_object *obj;
1719 uint32_t read_domains = args->read_domains;
1720 uint32_t write_domain = args->write_domain;
1721 int err;
1722
1723
1724 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1725 return -EINVAL;
1726
1727
1728
1729
1730 if (write_domain != 0 && read_domains != write_domain)
1731 return -EINVAL;
1732
1733 obj = i915_gem_object_lookup(file, args->handle);
1734 if (!obj)
1735 return -ENOENT;
1736
1737
1738
1739
1740
1741 err = i915_gem_object_wait(obj,
1742 I915_WAIT_INTERRUPTIBLE |
1743 (write_domain ? I915_WAIT_ALL : 0),
1744 MAX_SCHEDULE_TIMEOUT,
1745 to_rps_client(file));
1746 if (err)
1747 goto out;
1748
1749
1750
1751
1752
1753
1754
1755 if (i915_gem_object_is_proxy(obj)) {
1756 err = -ENXIO;
1757 goto out;
1758 }
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769 err = i915_gem_object_pin_pages(obj);
1770 if (err)
1771 goto out;
1772
1773 err = i915_mutex_lock_interruptible(dev);
1774 if (err)
1775 goto out_unpin;
1776
1777 if (read_domains & I915_GEM_DOMAIN_WC)
1778 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
1779 else if (read_domains & I915_GEM_DOMAIN_GTT)
1780 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
1781 else
1782 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
1783
1784
1785 i915_gem_object_bump_inactive_ggtt(obj);
1786
1787 mutex_unlock(&dev->struct_mutex);
1788
1789 if (write_domain != 0)
1790 intel_fb_obj_invalidate(obj,
1791 fb_write_origin(obj, write_domain));
1792
1793out_unpin:
1794 i915_gem_object_unpin_pages(obj);
1795out:
1796 i915_gem_object_put(obj);
1797 return err;
1798}
1799
1800
1801
1802
1803
1804
1805
1806int
1807i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1808 struct drm_file *file)
1809{
1810 struct drm_i915_gem_sw_finish *args = data;
1811 struct drm_i915_gem_object *obj;
1812
1813 obj = i915_gem_object_lookup(file, args->handle);
1814 if (!obj)
1815 return -ENOENT;
1816
1817
1818
1819
1820
1821
1822
1823 i915_gem_object_flush_if_display(obj);
1824 i915_gem_object_put(obj);
1825
1826 return 0;
1827}
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849int
1850i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1851 struct drm_file *file)
1852{
1853 struct drm_i915_gem_mmap *args = data;
1854 struct drm_i915_gem_object *obj;
1855 unsigned long addr;
1856
1857 if (args->flags & ~(I915_MMAP_WC))
1858 return -EINVAL;
1859
1860 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1861 return -ENODEV;
1862
1863 obj = i915_gem_object_lookup(file, args->handle);
1864 if (!obj)
1865 return -ENOENT;
1866
1867
1868
1869
1870 if (!obj->base.filp) {
1871 i915_gem_object_put(obj);
1872 return -ENXIO;
1873 }
1874
1875 addr = vm_mmap(obj->base.filp, 0, args->size,
1876 PROT_READ | PROT_WRITE, MAP_SHARED,
1877 args->offset);
1878 if (args->flags & I915_MMAP_WC) {
1879 struct mm_struct *mm = current->mm;
1880 struct vm_area_struct *vma;
1881
1882 if (down_write_killable(&mm->mmap_sem)) {
1883 i915_gem_object_put(obj);
1884 return -EINTR;
1885 }
1886 vma = find_vma(mm, addr);
1887 if (vma)
1888 vma->vm_page_prot =
1889 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1890 else
1891 addr = -ENOMEM;
1892 up_write(&mm->mmap_sem);
1893
1894
1895 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
1896 }
1897 i915_gem_object_put(obj);
1898 if (IS_ERR((void *)addr))
1899 return addr;
1900
1901 args->addr_ptr = (uint64_t) addr;
1902
1903 return 0;
1904}
1905
1906static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
1907{
1908 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
1909}
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959int i915_gem_mmap_gtt_version(void)
1960{
1961 return 2;
1962}
1963
1964static inline struct i915_ggtt_view
1965compute_partial_view(const struct drm_i915_gem_object *obj,
1966 pgoff_t page_offset,
1967 unsigned int chunk)
1968{
1969 struct i915_ggtt_view view;
1970
1971 if (i915_gem_object_is_tiled(obj))
1972 chunk = roundup(chunk, tile_row_pages(obj));
1973
1974 view.type = I915_GGTT_VIEW_PARTIAL;
1975 view.partial.offset = rounddown(page_offset, chunk);
1976 view.partial.size =
1977 min_t(unsigned int, chunk,
1978 (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
1979
1980
1981 if (chunk >= obj->base.size >> PAGE_SHIFT)
1982 view.type = I915_GGTT_VIEW_NORMAL;
1983
1984 return view;
1985}
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005vm_fault_t i915_gem_fault(struct vm_fault *vmf)
2006{
2007#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
2008 struct vm_area_struct *area = vmf->vma;
2009 struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
2010 struct drm_device *dev = obj->base.dev;
2011 struct drm_i915_private *dev_priv = to_i915(dev);
2012 struct i915_ggtt *ggtt = &dev_priv->ggtt;
2013 bool write = area->vm_flags & VM_WRITE;
2014 struct i915_vma *vma;
2015 pgoff_t page_offset;
2016 int ret;
2017
2018
2019 if (i915_gem_object_is_readonly(obj) && write)
2020 return VM_FAULT_SIGBUS;
2021
2022
2023 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
2024
2025 trace_i915_gem_object_fault(obj, page_offset, true, write);
2026
2027
2028
2029
2030
2031
2032 ret = i915_gem_object_wait(obj,
2033 I915_WAIT_INTERRUPTIBLE,
2034 MAX_SCHEDULE_TIMEOUT,
2035 NULL);
2036 if (ret)
2037 goto err;
2038
2039 ret = i915_gem_object_pin_pages(obj);
2040 if (ret)
2041 goto err;
2042
2043 intel_runtime_pm_get(dev_priv);
2044
2045 ret = i915_mutex_lock_interruptible(dev);
2046 if (ret)
2047 goto err_rpm;
2048
2049
2050 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
2051 ret = -EFAULT;
2052 goto err_unlock;
2053 }
2054
2055
2056
2057 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
2058 PIN_MAPPABLE |
2059 PIN_NONBLOCK |
2060 PIN_NONFAULT);
2061 if (IS_ERR(vma)) {
2062
2063 struct i915_ggtt_view view =
2064 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
2065 unsigned int flags;
2066
2067 flags = PIN_MAPPABLE;
2068 if (view.type == I915_GGTT_VIEW_NORMAL)
2069 flags |= PIN_NONBLOCK;
2070
2071
2072
2073
2074
2075 obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
2076
2077 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
2078 if (IS_ERR(vma) && !view.type) {
2079 flags = PIN_MAPPABLE;
2080 view.type = I915_GGTT_VIEW_PARTIAL;
2081 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
2082 }
2083 }
2084 if (IS_ERR(vma)) {
2085 ret = PTR_ERR(vma);
2086 goto err_unlock;
2087 }
2088
2089 ret = i915_gem_object_set_to_gtt_domain(obj, write);
2090 if (ret)
2091 goto err_unpin;
2092
2093 ret = i915_vma_pin_fence(vma);
2094 if (ret)
2095 goto err_unpin;
2096
2097
2098 ret = remap_io_mapping(area,
2099 area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
2100 (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
2101 min_t(u64, vma->size, area->vm_end - area->vm_start),
2102 &ggtt->iomap);
2103 if (ret)
2104 goto err_fence;
2105
2106
2107 assert_rpm_wakelock_held(dev_priv);
2108 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
2109 list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
2110 GEM_BUG_ON(!obj->userfault_count);
2111
2112 i915_vma_set_ggtt_write(vma);
2113
2114err_fence:
2115 i915_vma_unpin_fence(vma);
2116err_unpin:
2117 __i915_vma_unpin(vma);
2118err_unlock:
2119 mutex_unlock(&dev->struct_mutex);
2120err_rpm:
2121 intel_runtime_pm_put(dev_priv);
2122 i915_gem_object_unpin_pages(obj);
2123err:
2124 switch (ret) {
2125 case -EIO:
2126
2127
2128
2129
2130
2131
2132 if (!i915_terminally_wedged(&dev_priv->gpu_error))
2133 return VM_FAULT_SIGBUS;
2134
2135 case -EAGAIN:
2136
2137
2138
2139
2140
2141 case 0:
2142 case -ERESTARTSYS:
2143 case -EINTR:
2144 case -EBUSY:
2145
2146
2147
2148
2149 return VM_FAULT_NOPAGE;
2150 case -ENOMEM:
2151 return VM_FAULT_OOM;
2152 case -ENOSPC:
2153 case -EFAULT:
2154 return VM_FAULT_SIGBUS;
2155 default:
2156 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
2157 return VM_FAULT_SIGBUS;
2158 }
2159}
2160
2161static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
2162{
2163 struct i915_vma *vma;
2164
2165 GEM_BUG_ON(!obj->userfault_count);
2166
2167 obj->userfault_count = 0;
2168 list_del(&obj->userfault_link);
2169 drm_vma_node_unmap(&obj->base.vma_node,
2170 obj->base.dev->anon_inode->i_mapping);
2171
2172 for_each_ggtt_vma(vma, obj)
2173 i915_vma_unset_userfault(vma);
2174}
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190void
2191i915_gem_release_mmap(struct drm_i915_gem_object *obj)
2192{
2193 struct drm_i915_private *i915 = to_i915(obj->base.dev);
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203 lockdep_assert_held(&i915->drm.struct_mutex);
2204 intel_runtime_pm_get(i915);
2205
2206 if (!obj->userfault_count)
2207 goto out;
2208
2209 __i915_gem_object_release_mmap(obj);
2210
2211
2212
2213
2214
2215
2216
2217
2218 wmb();
2219
2220out:
2221 intel_runtime_pm_put(i915);
2222}
2223
2224void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
2225{
2226 struct drm_i915_gem_object *obj, *on;
2227 int i;
2228
2229
2230
2231
2232
2233
2234
2235
2236 list_for_each_entry_safe(obj, on,
2237 &dev_priv->mm.userfault_list, userfault_link)
2238 __i915_gem_object_release_mmap(obj);
2239
2240
2241
2242
2243
2244 for (i = 0; i < dev_priv->num_fence_regs; i++) {
2245 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258 if (!reg->vma)
2259 continue;
2260
2261 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
2262 reg->dirty = true;
2263 }
2264}
2265
2266static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2267{
2268 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2269 int err;
2270
2271 err = drm_gem_create_mmap_offset(&obj->base);
2272 if (likely(!err))
2273 return 0;
2274
2275
2276 do {
2277 err = i915_gem_wait_for_idle(dev_priv,
2278 I915_WAIT_INTERRUPTIBLE,
2279 MAX_SCHEDULE_TIMEOUT);
2280 if (err)
2281 break;
2282
2283 i915_gem_drain_freed_objects(dev_priv);
2284 err = drm_gem_create_mmap_offset(&obj->base);
2285 if (!err)
2286 break;
2287
2288 } while (flush_delayed_work(&dev_priv->gt.retire_work));
2289
2290 return err;
2291}
2292
2293static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2294{
2295 drm_gem_free_mmap_offset(&obj->base);
2296}
2297
2298int
2299i915_gem_mmap_gtt(struct drm_file *file,
2300 struct drm_device *dev,
2301 uint32_t handle,
2302 uint64_t *offset)
2303{
2304 struct drm_i915_gem_object *obj;
2305 int ret;
2306
2307 obj = i915_gem_object_lookup(file, handle);
2308 if (!obj)
2309 return -ENOENT;
2310
2311 ret = i915_gem_object_create_mmap_offset(obj);
2312 if (ret == 0)
2313 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2314
2315 i915_gem_object_put(obj);
2316 return ret;
2317}
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334int
2335i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2336 struct drm_file *file)
2337{
2338 struct drm_i915_gem_mmap_gtt *args = data;
2339
2340 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2341}
2342
2343
2344static void
2345i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2346{
2347 i915_gem_object_free_mmap_offset(obj);
2348
2349 if (obj->base.filp == NULL)
2350 return;
2351
2352
2353
2354
2355
2356
2357 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2358 obj->mm.madv = __I915_MADV_PURGED;
2359 obj->mm.pages = ERR_PTR(-EFAULT);
2360}
2361
2362
2363void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2364{
2365 struct address_space *mapping;
2366
2367 lockdep_assert_held(&obj->mm.lock);
2368 GEM_BUG_ON(i915_gem_object_has_pages(obj));
2369
2370 switch (obj->mm.madv) {
2371 case I915_MADV_DONTNEED:
2372 i915_gem_object_truncate(obj);
2373 case __I915_MADV_PURGED:
2374 return;
2375 }
2376
2377 if (obj->base.filp == NULL)
2378 return;
2379
2380 mapping = obj->base.filp->f_mapping,
2381 invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2382}
2383
2384static void
2385i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
2386 struct sg_table *pages)
2387{
2388 struct sgt_iter sgt_iter;
2389 struct page *page;
2390
2391 __i915_gem_object_release_shmem(obj, pages, true);
2392
2393 i915_gem_gtt_finish_pages(obj, pages);
2394
2395 if (i915_gem_object_needs_bit17_swizzle(obj))
2396 i915_gem_object_save_bit_17_swizzle(obj, pages);
2397
2398 for_each_sgt_page(page, sgt_iter, pages) {
2399 if (obj->mm.dirty)
2400 set_page_dirty(page);
2401
2402 if (obj->mm.madv == I915_MADV_WILLNEED)
2403 mark_page_accessed(page);
2404
2405 put_page(page);
2406 }
2407 obj->mm.dirty = false;
2408
2409 sg_free_table(pages);
2410 kfree(pages);
2411}
2412
2413static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
2414{
2415 struct radix_tree_iter iter;
2416 void __rcu **slot;
2417
2418 rcu_read_lock();
2419 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
2420 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
2421 rcu_read_unlock();
2422}
2423
2424static struct sg_table *
2425__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
2426{
2427 struct drm_i915_private *i915 = to_i915(obj->base.dev);
2428 struct sg_table *pages;
2429
2430 pages = fetch_and_zero(&obj->mm.pages);
2431 if (!pages)
2432 return NULL;
2433
2434 spin_lock(&i915->mm.obj_lock);
2435 list_del(&obj->mm.link);
2436 spin_unlock(&i915->mm.obj_lock);
2437
2438 if (obj->mm.mapping) {
2439 void *ptr;
2440
2441 ptr = page_mask_bits(obj->mm.mapping);
2442 if (is_vmalloc_addr(ptr))
2443 vunmap(ptr);
2444 else
2445 kunmap(kmap_to_page(ptr));
2446
2447 obj->mm.mapping = NULL;
2448 }
2449
2450 __i915_gem_object_reset_page_iter(obj);
2451 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
2452
2453 return pages;
2454}
2455
2456void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
2457 enum i915_mm_subclass subclass)
2458{
2459 struct sg_table *pages;
2460
2461 if (i915_gem_object_has_pinned_pages(obj))
2462 return;
2463
2464 GEM_BUG_ON(obj->bind_count);
2465 if (!i915_gem_object_has_pages(obj))
2466 return;
2467
2468
2469 mutex_lock_nested(&obj->mm.lock, subclass);
2470 if (unlikely(atomic_read(&obj->mm.pages_pin_count)))
2471 goto unlock;
2472
2473
2474
2475
2476
2477
2478 pages = __i915_gem_object_unset_pages(obj);
2479 if (!IS_ERR(pages))
2480 obj->ops->put_pages(obj, pages);
2481
2482unlock:
2483 mutex_unlock(&obj->mm.lock);
2484}
2485
2486static bool i915_sg_trim(struct sg_table *orig_st)
2487{
2488 struct sg_table new_st;
2489 struct scatterlist *sg, *new_sg;
2490 unsigned int i;
2491
2492 if (orig_st->nents == orig_st->orig_nents)
2493 return false;
2494
2495 if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
2496 return false;
2497
2498 new_sg = new_st.sgl;
2499 for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
2500 sg_set_page(new_sg, sg_page(sg), sg->length, 0);
2501 sg_dma_address(new_sg) = sg_dma_address(sg);
2502 sg_dma_len(new_sg) = sg_dma_len(sg);
2503
2504 new_sg = sg_next(new_sg);
2505 }
2506 GEM_BUG_ON(new_sg);
2507
2508 sg_free_table(orig_st);
2509
2510 *orig_st = new_st;
2511 return true;
2512}
2513
2514static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2515{
2516 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2517 const unsigned long page_count = obj->base.size / PAGE_SIZE;
2518 unsigned long i;
2519 struct address_space *mapping;
2520 struct sg_table *st;
2521 struct scatterlist *sg;
2522 struct sgt_iter sgt_iter;
2523 struct page *page;
2524 unsigned long last_pfn = 0;
2525 unsigned int max_segment = i915_sg_segment_size();
2526 unsigned int sg_page_sizes;
2527 gfp_t noreclaim;
2528 int ret;
2529
2530
2531
2532
2533
2534
2535 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2536 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2537
2538
2539
2540
2541
2542 if (page_count > totalram_pages)
2543 return -ENOMEM;
2544
2545 st = kmalloc(sizeof(*st), GFP_KERNEL);
2546 if (st == NULL)
2547 return -ENOMEM;
2548
2549rebuild_st:
2550 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2551 kfree(st);
2552 return -ENOMEM;
2553 }
2554
2555
2556
2557
2558
2559
2560
2561 mapping = obj->base.filp->f_mapping;
2562 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
2563 noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
2564
2565 sg = st->sgl;
2566 st->nents = 0;
2567 sg_page_sizes = 0;
2568 for (i = 0; i < page_count; i++) {
2569 const unsigned int shrink[] = {
2570 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE,
2571 0,
2572 }, *s = shrink;
2573 gfp_t gfp = noreclaim;
2574
2575 do {
2576 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2577 if (likely(!IS_ERR(page)))
2578 break;
2579
2580 if (!*s) {
2581 ret = PTR_ERR(page);
2582 goto err_sg;
2583 }
2584
2585 i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++);
2586 cond_resched();
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597 if (!*s) {
2598
2599 gfp = mapping_gfp_mask(mapping);
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615 gfp |= __GFP_RETRY_MAYFAIL;
2616 }
2617 } while (1);
2618
2619 if (!i ||
2620 sg->length >= max_segment ||
2621 page_to_pfn(page) != last_pfn + 1) {
2622 if (i) {
2623 sg_page_sizes |= sg->length;
2624 sg = sg_next(sg);
2625 }
2626 st->nents++;
2627 sg_set_page(sg, page, PAGE_SIZE, 0);
2628 } else {
2629 sg->length += PAGE_SIZE;
2630 }
2631 last_pfn = page_to_pfn(page);
2632
2633
2634 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2635 }
2636 if (sg) {
2637 sg_page_sizes |= sg->length;
2638 sg_mark_end(sg);
2639 }
2640
2641
2642 i915_sg_trim(st);
2643
2644 ret = i915_gem_gtt_prepare_pages(obj, st);
2645 if (ret) {
2646
2647
2648
2649
2650
2651 if (max_segment > PAGE_SIZE) {
2652 for_each_sgt_page(page, sgt_iter, st)
2653 put_page(page);
2654 sg_free_table(st);
2655
2656 max_segment = PAGE_SIZE;
2657 goto rebuild_st;
2658 } else {
2659 dev_warn(&dev_priv->drm.pdev->dev,
2660 "Failed to DMA remap %lu pages\n",
2661 page_count);
2662 goto err_pages;
2663 }
2664 }
2665
2666 if (i915_gem_object_needs_bit17_swizzle(obj))
2667 i915_gem_object_do_bit_17_swizzle(obj, st);
2668
2669 __i915_gem_object_set_pages(obj, st, sg_page_sizes);
2670
2671 return 0;
2672
2673err_sg:
2674 sg_mark_end(sg);
2675err_pages:
2676 for_each_sgt_page(page, sgt_iter, st)
2677 put_page(page);
2678 sg_free_table(st);
2679 kfree(st);
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690 if (ret == -ENOSPC)
2691 ret = -ENOMEM;
2692
2693 return ret;
2694}
2695
2696void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
2697 struct sg_table *pages,
2698 unsigned int sg_page_sizes)
2699{
2700 struct drm_i915_private *i915 = to_i915(obj->base.dev);
2701 unsigned long supported = INTEL_INFO(i915)->page_sizes;
2702 int i;
2703
2704 lockdep_assert_held(&obj->mm.lock);
2705
2706 obj->mm.get_page.sg_pos = pages->sgl;
2707 obj->mm.get_page.sg_idx = 0;
2708
2709 obj->mm.pages = pages;
2710
2711 if (i915_gem_object_is_tiled(obj) &&
2712 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
2713 GEM_BUG_ON(obj->mm.quirked);
2714 __i915_gem_object_pin_pages(obj);
2715 obj->mm.quirked = true;
2716 }
2717
2718 GEM_BUG_ON(!sg_page_sizes);
2719 obj->mm.page_sizes.phys = sg_page_sizes;
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729 obj->mm.page_sizes.sg = 0;
2730 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
2731 if (obj->mm.page_sizes.phys & ~0u << i)
2732 obj->mm.page_sizes.sg |= BIT(i);
2733 }
2734 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
2735
2736 spin_lock(&i915->mm.obj_lock);
2737 list_add(&obj->mm.link, &i915->mm.unbound_list);
2738 spin_unlock(&i915->mm.obj_lock);
2739}
2740
2741static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2742{
2743 int err;
2744
2745 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
2746 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2747 return -EFAULT;
2748 }
2749
2750 err = obj->ops->get_pages(obj);
2751 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
2752
2753 return err;
2754}
2755
2756
2757
2758
2759
2760
2761
2762
2763int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2764{
2765 int err;
2766
2767 err = mutex_lock_interruptible(&obj->mm.lock);
2768 if (err)
2769 return err;
2770
2771 if (unlikely(!i915_gem_object_has_pages(obj))) {
2772 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2773
2774 err = ____i915_gem_object_get_pages(obj);
2775 if (err)
2776 goto unlock;
2777
2778 smp_mb__before_atomic();
2779 }
2780 atomic_inc(&obj->mm.pages_pin_count);
2781
2782unlock:
2783 mutex_unlock(&obj->mm.lock);
2784 return err;
2785}
2786
2787
2788static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2789 enum i915_map_type type)
2790{
2791 unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2792 struct sg_table *sgt = obj->mm.pages;
2793 struct sgt_iter sgt_iter;
2794 struct page *page;
2795 struct page *stack_pages[32];
2796 struct page **pages = stack_pages;
2797 unsigned long i = 0;
2798 pgprot_t pgprot;
2799 void *addr;
2800
2801
2802 if (n_pages == 1 && type == I915_MAP_WB)
2803 return kmap(sg_page(sgt->sgl));
2804
2805 if (n_pages > ARRAY_SIZE(stack_pages)) {
2806
2807 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
2808 if (!pages)
2809 return NULL;
2810 }
2811
2812 for_each_sgt_page(page, sgt_iter, sgt)
2813 pages[i++] = page;
2814
2815
2816 GEM_BUG_ON(i != n_pages);
2817
2818 switch (type) {
2819 default:
2820 MISSING_CASE(type);
2821
2822 case I915_MAP_WB:
2823 pgprot = PAGE_KERNEL;
2824 break;
2825 case I915_MAP_WC:
2826 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2827 break;
2828 }
2829 addr = vmap(pages, n_pages, 0, pgprot);
2830
2831 if (pages != stack_pages)
2832 kvfree(pages);
2833
2834 return addr;
2835}
2836
2837
2838void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2839 enum i915_map_type type)
2840{
2841 enum i915_map_type has_type;
2842 bool pinned;
2843 void *ptr;
2844 int ret;
2845
2846 if (unlikely(!i915_gem_object_has_struct_page(obj)))
2847 return ERR_PTR(-ENXIO);
2848
2849 ret = mutex_lock_interruptible(&obj->mm.lock);
2850 if (ret)
2851 return ERR_PTR(ret);
2852
2853 pinned = !(type & I915_MAP_OVERRIDE);
2854 type &= ~I915_MAP_OVERRIDE;
2855
2856 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
2857 if (unlikely(!i915_gem_object_has_pages(obj))) {
2858 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2859
2860 ret = ____i915_gem_object_get_pages(obj);
2861 if (ret)
2862 goto err_unlock;
2863
2864 smp_mb__before_atomic();
2865 }
2866 atomic_inc(&obj->mm.pages_pin_count);
2867 pinned = false;
2868 }
2869 GEM_BUG_ON(!i915_gem_object_has_pages(obj));
2870
2871 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
2872 if (ptr && has_type != type) {
2873 if (pinned) {
2874 ret = -EBUSY;
2875 goto err_unpin;
2876 }
2877
2878 if (is_vmalloc_addr(ptr))
2879 vunmap(ptr);
2880 else
2881 kunmap(kmap_to_page(ptr));
2882
2883 ptr = obj->mm.mapping = NULL;
2884 }
2885
2886 if (!ptr) {
2887 ptr = i915_gem_object_map(obj, type);
2888 if (!ptr) {
2889 ret = -ENOMEM;
2890 goto err_unpin;
2891 }
2892
2893 obj->mm.mapping = page_pack_bits(ptr, type);
2894 }
2895
2896out_unlock:
2897 mutex_unlock(&obj->mm.lock);
2898 return ptr;
2899
2900err_unpin:
2901 atomic_dec(&obj->mm.pages_pin_count);
2902err_unlock:
2903 ptr = ERR_PTR(ret);
2904 goto out_unlock;
2905}
2906
2907static int
2908i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
2909 const struct drm_i915_gem_pwrite *arg)
2910{
2911 struct address_space *mapping = obj->base.filp->f_mapping;
2912 char __user *user_data = u64_to_user_ptr(arg->data_ptr);
2913 u64 remain, offset;
2914 unsigned int pg;
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924 if (i915_gem_object_has_pages(obj))
2925 return -ENODEV;
2926
2927 if (obj->mm.madv != I915_MADV_WILLNEED)
2928 return -EFAULT;
2929
2930
2931
2932
2933
2934
2935
2936
2937 remain = arg->size;
2938 offset = arg->offset;
2939 pg = offset_in_page(offset);
2940
2941 do {
2942 unsigned int len, unwritten;
2943 struct page *page;
2944 void *data, *vaddr;
2945 int err;
2946
2947 len = PAGE_SIZE - pg;
2948 if (len > remain)
2949 len = remain;
2950
2951 err = pagecache_write_begin(obj->base.filp, mapping,
2952 offset, len, 0,
2953 &page, &data);
2954 if (err < 0)
2955 return err;
2956
2957 vaddr = kmap(page);
2958 unwritten = copy_from_user(vaddr + pg, user_data, len);
2959 kunmap(page);
2960
2961 err = pagecache_write_end(obj->base.filp, mapping,
2962 offset, len, len - unwritten,
2963 page, data);
2964 if (err < 0)
2965 return err;
2966
2967 if (unwritten)
2968 return -EFAULT;
2969
2970 remain -= len;
2971 user_data += len;
2972 offset += len;
2973 pg = 0;
2974 } while (remain);
2975
2976 return 0;
2977}
2978
2979static void i915_gem_client_mark_guilty(struct drm_i915_file_private *file_priv,
2980 const struct i915_gem_context *ctx)
2981{
2982 unsigned int score;
2983 unsigned long prev_hang;
2984
2985 if (i915_gem_context_is_banned(ctx))
2986 score = I915_CLIENT_SCORE_CONTEXT_BAN;
2987 else
2988 score = 0;
2989
2990 prev_hang = xchg(&file_priv->hang_timestamp, jiffies);
2991 if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES))
2992 score += I915_CLIENT_SCORE_HANG_FAST;
2993
2994 if (score) {
2995 atomic_add(score, &file_priv->ban_score);
2996
2997 DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n",
2998 ctx->name, score,
2999 atomic_read(&file_priv->ban_score));
3000 }
3001}
3002
3003static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx)
3004{
3005 unsigned int score;
3006 bool banned, bannable;
3007
3008 atomic_inc(&ctx->guilty_count);
3009
3010 bannable = i915_gem_context_is_bannable(ctx);
3011 score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score);
3012 banned = score >= CONTEXT_SCORE_BAN_THRESHOLD;
3013
3014
3015 if (!bannable)
3016 return;
3017
3018 if (banned) {
3019 DRM_DEBUG_DRIVER("context %s: guilty %d, score %u, banned\n",
3020 ctx->name, atomic_read(&ctx->guilty_count),
3021 score);
3022 i915_gem_context_set_banned(ctx);
3023 }
3024
3025 if (!IS_ERR_OR_NULL(ctx->file_priv))
3026 i915_gem_client_mark_guilty(ctx->file_priv, ctx);
3027}
3028
3029static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx)
3030{
3031 atomic_inc(&ctx->active_count);
3032}
3033
3034struct i915_request *
3035i915_gem_find_active_request(struct intel_engine_cs *engine)
3036{
3037 struct i915_request *request, *active = NULL;
3038 unsigned long flags;
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051 spin_lock_irqsave(&engine->timeline.lock, flags);
3052 list_for_each_entry(request, &engine->timeline.requests, link) {
3053 if (__i915_request_completed(request, request->global_seqno))
3054 continue;
3055
3056 active = request;
3057 break;
3058 }
3059 spin_unlock_irqrestore(&engine->timeline.lock, flags);
3060
3061 return active;
3062}
3063
3064
3065
3066
3067
3068struct i915_request *
3069i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
3070{
3071 struct i915_request *request;
3072
3073
3074
3075
3076
3077
3078
3079
3080 intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);
3081
3082 request = engine->reset.prepare(engine);
3083 if (request && request->fence.error == -EIO)
3084 request = ERR_PTR(-EIO);
3085
3086 return request;
3087}
3088
3089int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
3090{
3091 struct intel_engine_cs *engine;
3092 struct i915_request *request;
3093 enum intel_engine_id id;
3094 int err = 0;
3095
3096 for_each_engine(engine, dev_priv, id) {
3097 request = i915_gem_reset_prepare_engine(engine);
3098 if (IS_ERR(request)) {
3099 err = PTR_ERR(request);
3100 continue;
3101 }
3102
3103 engine->hangcheck.active_request = request;
3104 }
3105
3106 i915_gem_revoke_fences(dev_priv);
3107 intel_uc_sanitize(dev_priv);
3108
3109 return err;
3110}
3111
3112static void engine_skip_context(struct i915_request *request)
3113{
3114 struct intel_engine_cs *engine = request->engine;
3115 struct i915_gem_context *hung_ctx = request->gem_context;
3116 struct i915_timeline *timeline = request->timeline;
3117 unsigned long flags;
3118
3119 GEM_BUG_ON(timeline == &engine->timeline);
3120
3121 spin_lock_irqsave(&engine->timeline.lock, flags);
3122 spin_lock(&timeline->lock);
3123
3124 list_for_each_entry_continue(request, &engine->timeline.requests, link)
3125 if (request->gem_context == hung_ctx)
3126 i915_request_skip(request, -EIO);
3127
3128 list_for_each_entry(request, &timeline->requests, link)
3129 i915_request_skip(request, -EIO);
3130
3131 spin_unlock(&timeline->lock);
3132 spin_unlock_irqrestore(&engine->timeline.lock, flags);
3133}
3134
3135
3136static struct i915_request *
3137i915_gem_reset_request(struct intel_engine_cs *engine,
3138 struct i915_request *request,
3139 bool stalled)
3140{
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162 if (i915_request_completed(request)) {
3163 GEM_TRACE("%s pardoned global=%d (fence %llx:%d), current %d\n",
3164 engine->name, request->global_seqno,
3165 request->fence.context, request->fence.seqno,
3166 intel_engine_get_seqno(engine));
3167 stalled = false;
3168 }
3169
3170 if (stalled) {
3171 i915_gem_context_mark_guilty(request->gem_context);
3172 i915_request_skip(request, -EIO);
3173
3174
3175 if (i915_gem_context_is_banned(request->gem_context))
3176 engine_skip_context(request);
3177 } else {
3178
3179
3180
3181
3182
3183 request = i915_gem_find_active_request(engine);
3184 if (request) {
3185 unsigned long flags;
3186
3187 i915_gem_context_mark_innocent(request->gem_context);
3188 dma_fence_set_error(&request->fence, -EAGAIN);
3189
3190
3191 spin_lock_irqsave(&engine->timeline.lock, flags);
3192 request = list_prev_entry(request, link);
3193 if (&request->link == &engine->timeline.requests)
3194 request = NULL;
3195 spin_unlock_irqrestore(&engine->timeline.lock, flags);
3196 }
3197 }
3198
3199 return request;
3200}
3201
3202void i915_gem_reset_engine(struct intel_engine_cs *engine,
3203 struct i915_request *request,
3204 bool stalled)
3205{
3206
3207
3208
3209
3210
3211 smp_store_mb(engine->irq_posted, 0);
3212
3213 if (request)
3214 request = i915_gem_reset_request(engine, request, stalled);
3215
3216
3217 engine->reset.reset(engine, request);
3218}
3219
3220void i915_gem_reset(struct drm_i915_private *dev_priv,
3221 unsigned int stalled_mask)
3222{
3223 struct intel_engine_cs *engine;
3224 enum intel_engine_id id;
3225
3226 lockdep_assert_held(&dev_priv->drm.struct_mutex);
3227
3228 i915_retire_requests(dev_priv);
3229
3230 for_each_engine(engine, dev_priv, id) {
3231 struct intel_context *ce;
3232
3233 i915_gem_reset_engine(engine,
3234 engine->hangcheck.active_request,
3235 stalled_mask & ENGINE_MASK(id));
3236 ce = fetch_and_zero(&engine->last_retired_context);
3237 if (ce)
3238 intel_context_unpin(ce);
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250 if (intel_engine_is_idle(engine)) {
3251 struct i915_request *rq;
3252
3253 rq = i915_request_alloc(engine,
3254 dev_priv->kernel_context);
3255 if (!IS_ERR(rq))
3256 i915_request_add(rq);
3257 }
3258 }
3259
3260 i915_gem_restore_fences(dev_priv);
3261}
3262
3263void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
3264{
3265 engine->reset.finish(engine);
3266
3267 intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
3268}
3269
3270void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
3271{
3272 struct intel_engine_cs *engine;
3273 enum intel_engine_id id;
3274
3275 lockdep_assert_held(&dev_priv->drm.struct_mutex);
3276
3277 for_each_engine(engine, dev_priv, id) {
3278 engine->hangcheck.active_request = NULL;
3279 i915_gem_reset_finish_engine(engine);
3280 }
3281}
3282
3283static void nop_submit_request(struct i915_request *request)
3284{
3285 GEM_TRACE("%s fence %llx:%d -> -EIO\n",
3286 request->engine->name,
3287 request->fence.context, request->fence.seqno);
3288 dma_fence_set_error(&request->fence, -EIO);
3289
3290 i915_request_submit(request);
3291}
3292
3293static void nop_complete_submit_request(struct i915_request *request)
3294{
3295 unsigned long flags;
3296
3297 GEM_TRACE("%s fence %llx:%d -> -EIO\n",
3298 request->engine->name,
3299 request->fence.context, request->fence.seqno);
3300 dma_fence_set_error(&request->fence, -EIO);
3301
3302 spin_lock_irqsave(&request->engine->timeline.lock, flags);
3303 __i915_request_submit(request);
3304 intel_engine_init_global_seqno(request->engine, request->global_seqno);
3305 spin_unlock_irqrestore(&request->engine->timeline.lock, flags);
3306}
3307
3308void i915_gem_set_wedged(struct drm_i915_private *i915)
3309{
3310 struct intel_engine_cs *engine;
3311 enum intel_engine_id id;
3312
3313 GEM_TRACE("start\n");
3314
3315 if (GEM_SHOW_DEBUG()) {
3316 struct drm_printer p = drm_debug_printer(__func__);
3317
3318 for_each_engine(engine, i915, id)
3319 intel_engine_dump(engine, &p, "%s\n", engine->name);
3320 }
3321
3322 if (test_and_set_bit(I915_WEDGED, &i915->gpu_error.flags))
3323 goto out;
3324
3325
3326
3327
3328
3329
3330 for_each_engine(engine, i915, id) {
3331 i915_gem_reset_prepare_engine(engine);
3332
3333 engine->submit_request = nop_submit_request;
3334 engine->schedule = NULL;
3335 }
3336 i915->caps.scheduler = 0;
3337
3338
3339 if (INTEL_GEN(i915) >= 5)
3340 intel_gpu_reset(i915, ALL_ENGINES);
3341
3342
3343
3344
3345
3346
3347 synchronize_rcu();
3348
3349 for_each_engine(engine, i915, id) {
3350
3351 engine->cancel_requests(engine);
3352
3353
3354
3355
3356
3357 engine->submit_request = nop_complete_submit_request;
3358 }
3359
3360
3361
3362
3363
3364
3365 synchronize_rcu();
3366
3367 for_each_engine(engine, i915, id) {
3368 unsigned long flags;
3369
3370
3371
3372
3373
3374
3375 spin_lock_irqsave(&engine->timeline.lock, flags);
3376 intel_engine_init_global_seqno(engine,
3377 intel_engine_last_submit(engine));
3378 spin_unlock_irqrestore(&engine->timeline.lock, flags);
3379
3380 i915_gem_reset_finish_engine(engine);
3381 }
3382
3383out:
3384 GEM_TRACE("end\n");
3385
3386 wake_up_all(&i915->gpu_error.reset_queue);
3387}
3388
3389bool i915_gem_unset_wedged(struct drm_i915_private *i915)
3390{
3391 struct i915_timeline *tl;
3392
3393 lockdep_assert_held(&i915->drm.struct_mutex);
3394 if (!test_bit(I915_WEDGED, &i915->gpu_error.flags))
3395 return true;
3396
3397 GEM_TRACE("start\n");
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409 list_for_each_entry(tl, &i915->gt.timelines, link) {
3410 struct i915_request *rq;
3411
3412 rq = i915_gem_active_peek(&tl->last_request,
3413 &i915->drm.struct_mutex);
3414 if (!rq)
3415 continue;
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428 if (dma_fence_default_wait(&rq->fence, true,
3429 MAX_SCHEDULE_TIMEOUT) < 0)
3430 return false;
3431 }
3432 i915_retire_requests(i915);
3433 GEM_BUG_ON(i915->gt.active_requests);
3434
3435 if (!intel_gpu_reset(i915, ALL_ENGINES))
3436 intel_engines_sanitize(i915);
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447 intel_engines_reset_default_submission(i915);
3448 i915_gem_contexts_lost(i915);
3449
3450 GEM_TRACE("end\n");
3451
3452 smp_mb__before_atomic();
3453 clear_bit(I915_WEDGED, &i915->gpu_error.flags);
3454
3455 return true;
3456}
3457
3458static void
3459i915_gem_retire_work_handler(struct work_struct *work)
3460{
3461 struct drm_i915_private *dev_priv =
3462 container_of(work, typeof(*dev_priv), gt.retire_work.work);
3463 struct drm_device *dev = &dev_priv->drm;
3464
3465
3466 if (mutex_trylock(&dev->struct_mutex)) {
3467 i915_retire_requests(dev_priv);
3468 mutex_unlock(&dev->struct_mutex);
3469 }
3470
3471
3472
3473
3474
3475
3476 if (READ_ONCE(dev_priv->gt.awake))
3477 queue_delayed_work(dev_priv->wq,
3478 &dev_priv->gt.retire_work,
3479 round_jiffies_up_relative(HZ));
3480}
3481
3482static void shrink_caches(struct drm_i915_private *i915)
3483{
3484
3485
3486
3487
3488
3489 kmem_cache_shrink(i915->priorities);
3490 kmem_cache_shrink(i915->dependencies);
3491 kmem_cache_shrink(i915->requests);
3492 kmem_cache_shrink(i915->luts);
3493 kmem_cache_shrink(i915->vmas);
3494 kmem_cache_shrink(i915->objects);
3495}
3496
3497struct sleep_rcu_work {
3498 union {
3499 struct rcu_head rcu;
3500 struct work_struct work;
3501 };
3502 struct drm_i915_private *i915;
3503 unsigned int epoch;
3504};
3505
3506static inline bool
3507same_epoch(struct drm_i915_private *i915, unsigned int epoch)
3508{
3509
3510
3511
3512
3513
3514 return epoch == READ_ONCE(i915->gt.epoch);
3515}
3516
3517static void __sleep_work(struct work_struct *work)
3518{
3519 struct sleep_rcu_work *s = container_of(work, typeof(*s), work);
3520 struct drm_i915_private *i915 = s->i915;
3521 unsigned int epoch = s->epoch;
3522
3523 kfree(s);
3524 if (same_epoch(i915, epoch))
3525 shrink_caches(i915);
3526}
3527
3528static void __sleep_rcu(struct rcu_head *rcu)
3529{
3530 struct sleep_rcu_work *s = container_of(rcu, typeof(*s), rcu);
3531 struct drm_i915_private *i915 = s->i915;
3532
3533 if (same_epoch(i915, s->epoch)) {
3534 INIT_WORK(&s->work, __sleep_work);
3535 queue_work(i915->wq, &s->work);
3536 } else {
3537 kfree(s);
3538 }
3539}
3540
3541static inline bool
3542new_requests_since_last_retire(const struct drm_i915_private *i915)
3543{
3544 return (READ_ONCE(i915->gt.active_requests) ||
3545 work_pending(&i915->gt.idle_work.work));
3546}
3547
3548static void assert_kernel_context_is_current(struct drm_i915_private *i915)
3549{
3550 struct intel_engine_cs *engine;
3551 enum intel_engine_id id;
3552
3553 if (i915_terminally_wedged(&i915->gpu_error))
3554 return;
3555
3556 GEM_BUG_ON(i915->gt.active_requests);
3557 for_each_engine(engine, i915, id) {
3558 GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline.last_request));
3559 GEM_BUG_ON(engine->last_retired_context !=
3560 to_intel_context(i915->kernel_context, engine));
3561 }
3562}
3563
3564static void
3565i915_gem_idle_work_handler(struct work_struct *work)
3566{
3567 struct drm_i915_private *dev_priv =
3568 container_of(work, typeof(*dev_priv), gt.idle_work.work);
3569 unsigned int epoch = I915_EPOCH_INVALID;
3570 bool rearm_hangcheck;
3571
3572 if (!READ_ONCE(dev_priv->gt.awake))
3573 return;
3574
3575 if (READ_ONCE(dev_priv->gt.active_requests))
3576 return;
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586 mutex_lock(&dev_priv->drm.struct_mutex);
3587 i915_gem_switch_to_kernel_context(dev_priv);
3588 mutex_unlock(&dev_priv->drm.struct_mutex);
3589
3590 GEM_TRACE("active_requests=%d (after switch-to-kernel-context)\n",
3591 READ_ONCE(dev_priv->gt.active_requests));
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601 __wait_for(if (new_requests_since_last_retire(dev_priv)) return,
3602 intel_engines_are_idle(dev_priv),
3603 I915_IDLE_ENGINES_TIMEOUT * 1000,
3604 10, 500);
3605
3606 rearm_hangcheck =
3607 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
3608
3609 if (!mutex_trylock(&dev_priv->drm.struct_mutex)) {
3610
3611 mod_delayed_work(dev_priv->wq,
3612 &dev_priv->gt.idle_work,
3613 msecs_to_jiffies(50));
3614 goto out_rearm;
3615 }
3616
3617
3618
3619
3620
3621 if (new_requests_since_last_retire(dev_priv))
3622 goto out_unlock;
3623
3624 epoch = __i915_gem_park(dev_priv);
3625
3626 assert_kernel_context_is_current(dev_priv);
3627
3628 rearm_hangcheck = false;
3629out_unlock:
3630 mutex_unlock(&dev_priv->drm.struct_mutex);
3631
3632out_rearm:
3633 if (rearm_hangcheck) {
3634 GEM_BUG_ON(!dev_priv->gt.awake);
3635 i915_queue_hangcheck(dev_priv);
3636 }
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646 if (same_epoch(dev_priv, epoch)) {
3647 struct sleep_rcu_work *s = kmalloc(sizeof(*s), GFP_KERNEL);
3648 if (s) {
3649 s->i915 = dev_priv;
3650 s->epoch = epoch;
3651 call_rcu(&s->rcu, __sleep_rcu);
3652 }
3653 }
3654}
3655
3656void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
3657{
3658 struct drm_i915_private *i915 = to_i915(gem->dev);
3659 struct drm_i915_gem_object *obj = to_intel_bo(gem);
3660 struct drm_i915_file_private *fpriv = file->driver_priv;
3661 struct i915_lut_handle *lut, *ln;
3662
3663 mutex_lock(&i915->drm.struct_mutex);
3664
3665 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
3666 struct i915_gem_context *ctx = lut->ctx;
3667 struct i915_vma *vma;
3668
3669 GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF));
3670 if (ctx->file_priv != fpriv)
3671 continue;
3672
3673 vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
3674 GEM_BUG_ON(vma->obj != obj);
3675
3676
3677
3678
3679 GEM_BUG_ON(!vma->open_count);
3680 if (!--vma->open_count && !i915_vma_is_ggtt(vma))
3681 i915_vma_close(vma);
3682
3683 list_del(&lut->obj_link);
3684 list_del(&lut->ctx_link);
3685
3686 kmem_cache_free(i915->luts, lut);
3687 __i915_gem_object_release_unless_active(obj);
3688 }
3689
3690 mutex_unlock(&i915->drm.struct_mutex);
3691}
3692
3693static unsigned long to_wait_timeout(s64 timeout_ns)
3694{
3695 if (timeout_ns < 0)
3696 return MAX_SCHEDULE_TIMEOUT;
3697
3698 if (timeout_ns == 0)
3699 return 0;
3700
3701 return nsecs_to_jiffies_timeout(timeout_ns);
3702}
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728int
3729i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3730{
3731 struct drm_i915_gem_wait *args = data;
3732 struct drm_i915_gem_object *obj;
3733 ktime_t start;
3734 long ret;
3735
3736 if (args->flags != 0)
3737 return -EINVAL;
3738
3739 obj = i915_gem_object_lookup(file, args->bo_handle);
3740 if (!obj)
3741 return -ENOENT;
3742
3743 start = ktime_get();
3744
3745 ret = i915_gem_object_wait(obj,
3746 I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL,
3747 to_wait_timeout(args->timeout_ns),
3748 to_rps_client(file));
3749
3750 if (args->timeout_ns > 0) {
3751 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
3752 if (args->timeout_ns < 0)
3753 args->timeout_ns = 0;
3754
3755
3756
3757
3758
3759
3760
3761
3762 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
3763 args->timeout_ns = 0;
3764
3765
3766 if (ret == -ETIME && args->timeout_ns)
3767 ret = -EAGAIN;
3768 }
3769
3770 i915_gem_object_put(obj);
3771 return ret;
3772}
3773
3774static long wait_for_timeline(struct i915_timeline *tl,
3775 unsigned int flags, long timeout)
3776{
3777 struct i915_request *rq;
3778
3779 rq = i915_gem_active_get_unlocked(&tl->last_request);
3780 if (!rq)
3781 return timeout;
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792 if (flags & I915_WAIT_FOR_IDLE_BOOST)
3793 gen6_rps_boost(rq, NULL);
3794
3795 timeout = i915_request_wait(rq, flags, timeout);
3796 i915_request_put(rq);
3797
3798 return timeout;
3799}
3800
3801static int wait_for_engines(struct drm_i915_private *i915)
3802{
3803 if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) {
3804 dev_err(i915->drm.dev,
3805 "Failed to idle engines, declaring wedged!\n");
3806 GEM_TRACE_DUMP();
3807 i915_gem_set_wedged(i915);
3808 return -EIO;
3809 }
3810
3811 return 0;
3812}
3813
3814int i915_gem_wait_for_idle(struct drm_i915_private *i915,
3815 unsigned int flags, long timeout)
3816{
3817 GEM_TRACE("flags=%x (%s), timeout=%ld%s\n",
3818 flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked",
3819 timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "");
3820
3821
3822 if (!READ_ONCE(i915->gt.awake))
3823 return 0;
3824
3825 if (flags & I915_WAIT_LOCKED) {
3826 struct i915_timeline *tl;
3827 int err;
3828
3829 lockdep_assert_held(&i915->drm.struct_mutex);
3830
3831 list_for_each_entry(tl, &i915->gt.timelines, link) {
3832 timeout = wait_for_timeline(tl, flags, timeout);
3833 if (timeout < 0)
3834 return timeout;
3835 }
3836 if (GEM_SHOW_DEBUG() && !timeout) {
3837
3838 dev_warn(&i915->drm.pdev->dev,
3839 "Missed idle-completion interrupt!\n");
3840 GEM_TRACE_DUMP();
3841 }
3842
3843 err = wait_for_engines(i915);
3844 if (err)
3845 return err;
3846
3847 i915_retire_requests(i915);
3848 GEM_BUG_ON(i915->gt.active_requests);
3849 } else {
3850 struct intel_engine_cs *engine;
3851 enum intel_engine_id id;
3852
3853 for_each_engine(engine, i915, id) {
3854 struct i915_timeline *tl = &engine->timeline;
3855
3856 timeout = wait_for_timeline(tl, flags, timeout);
3857 if (timeout < 0)
3858 return timeout;
3859 }
3860 }
3861
3862 return 0;
3863}
3864
3865static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
3866{
3867
3868
3869
3870
3871 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
3872 if (obj->cache_dirty)
3873 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
3874 obj->write_domain = 0;
3875}
3876
3877void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
3878{
3879 if (!READ_ONCE(obj->pin_global))
3880 return;
3881
3882 mutex_lock(&obj->base.dev->struct_mutex);
3883 __i915_gem_object_flush_for_display(obj);
3884 mutex_unlock(&obj->base.dev->struct_mutex);
3885}
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895int
3896i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
3897{
3898 int ret;
3899
3900 lockdep_assert_held(&obj->base.dev->struct_mutex);
3901
3902 ret = i915_gem_object_wait(obj,
3903 I915_WAIT_INTERRUPTIBLE |
3904 I915_WAIT_LOCKED |
3905 (write ? I915_WAIT_ALL : 0),
3906 MAX_SCHEDULE_TIMEOUT,
3907 NULL);
3908 if (ret)
3909 return ret;
3910
3911 if (obj->write_domain == I915_GEM_DOMAIN_WC)
3912 return 0;
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922 ret = i915_gem_object_pin_pages(obj);
3923 if (ret)
3924 return ret;
3925
3926 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
3927
3928
3929
3930
3931
3932 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
3933 mb();
3934
3935
3936
3937
3938 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
3939 obj->read_domains |= I915_GEM_DOMAIN_WC;
3940 if (write) {
3941 obj->read_domains = I915_GEM_DOMAIN_WC;
3942 obj->write_domain = I915_GEM_DOMAIN_WC;
3943 obj->mm.dirty = true;
3944 }
3945
3946 i915_gem_object_unpin_pages(obj);
3947 return 0;
3948}
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958int
3959i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3960{
3961 int ret;
3962
3963 lockdep_assert_held(&obj->base.dev->struct_mutex);
3964
3965 ret = i915_gem_object_wait(obj,
3966 I915_WAIT_INTERRUPTIBLE |
3967 I915_WAIT_LOCKED |
3968 (write ? I915_WAIT_ALL : 0),
3969 MAX_SCHEDULE_TIMEOUT,
3970 NULL);
3971 if (ret)
3972 return ret;
3973
3974 if (obj->write_domain == I915_GEM_DOMAIN_GTT)
3975 return 0;
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985 ret = i915_gem_object_pin_pages(obj);
3986 if (ret)
3987 return ret;
3988
3989 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
3990
3991
3992
3993
3994
3995 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
3996 mb();
3997
3998
3999
4000
4001 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
4002 obj->read_domains |= I915_GEM_DOMAIN_GTT;
4003 if (write) {
4004 obj->read_domains = I915_GEM_DOMAIN_GTT;
4005 obj->write_domain = I915_GEM_DOMAIN_GTT;
4006 obj->mm.dirty = true;
4007 }
4008
4009 i915_gem_object_unpin_pages(obj);
4010 return 0;
4011}
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
4029 enum i915_cache_level cache_level)
4030{
4031 struct i915_vma *vma;
4032 int ret;
4033
4034 lockdep_assert_held(&obj->base.dev->struct_mutex);
4035
4036 if (obj->cache_level == cache_level)
4037 return 0;
4038
4039
4040
4041
4042
4043
4044restart:
4045 list_for_each_entry(vma, &obj->vma_list, obj_link) {
4046 if (!drm_mm_node_allocated(&vma->node))
4047 continue;
4048
4049 if (i915_vma_is_pinned(vma)) {
4050 DRM_DEBUG("can not change the cache level of pinned objects\n");
4051 return -EBUSY;
4052 }
4053
4054 if (!i915_vma_is_closed(vma) &&
4055 i915_gem_valid_gtt_space(vma, cache_level))
4056 continue;
4057
4058 ret = i915_vma_unbind(vma);
4059 if (ret)
4060 return ret;
4061
4062
4063
4064
4065
4066 goto restart;
4067 }
4068
4069
4070
4071
4072
4073
4074
4075
4076 if (obj->bind_count) {
4077
4078
4079
4080
4081 ret = i915_gem_object_wait(obj,
4082 I915_WAIT_INTERRUPTIBLE |
4083 I915_WAIT_LOCKED |
4084 I915_WAIT_ALL,
4085 MAX_SCHEDULE_TIMEOUT,
4086 NULL);
4087 if (ret)
4088 return ret;
4089
4090 if (!HAS_LLC(to_i915(obj->base.dev)) &&
4091 cache_level != I915_CACHE_NONE) {
4092
4093
4094
4095
4096
4097
4098
4099 i915_gem_release_mmap(obj);
4100
4101
4102
4103
4104
4105
4106
4107
4108 for_each_ggtt_vma(vma, obj) {
4109 ret = i915_vma_put_fence(vma);
4110 if (ret)
4111 return ret;
4112 }
4113 } else {
4114
4115
4116
4117
4118
4119
4120
4121 }
4122
4123 list_for_each_entry(vma, &obj->vma_list, obj_link) {
4124 if (!drm_mm_node_allocated(&vma->node))
4125 continue;
4126
4127 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
4128 if (ret)
4129 return ret;
4130 }
4131 }
4132
4133 list_for_each_entry(vma, &obj->vma_list, obj_link)
4134 vma->node.color = cache_level;
4135 i915_gem_object_set_cache_coherency(obj, cache_level);
4136 obj->cache_dirty = true;
4137
4138 return 0;
4139}
4140
4141int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
4142 struct drm_file *file)
4143{
4144 struct drm_i915_gem_caching *args = data;
4145 struct drm_i915_gem_object *obj;
4146 int err = 0;
4147
4148 rcu_read_lock();
4149 obj = i915_gem_object_lookup_rcu(file, args->handle);
4150 if (!obj) {
4151 err = -ENOENT;
4152 goto out;
4153 }
4154
4155 switch (obj->cache_level) {
4156 case I915_CACHE_LLC:
4157 case I915_CACHE_L3_LLC:
4158 args->caching = I915_CACHING_CACHED;
4159 break;
4160
4161 case I915_CACHE_WT:
4162 args->caching = I915_CACHING_DISPLAY;
4163 break;
4164
4165 default:
4166 args->caching = I915_CACHING_NONE;
4167 break;
4168 }
4169out:
4170 rcu_read_unlock();
4171 return err;
4172}
4173
4174int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
4175 struct drm_file *file)
4176{
4177 struct drm_i915_private *i915 = to_i915(dev);
4178 struct drm_i915_gem_caching *args = data;
4179 struct drm_i915_gem_object *obj;
4180 enum i915_cache_level level;
4181 int ret = 0;
4182
4183 switch (args->caching) {
4184 case I915_CACHING_NONE:
4185 level = I915_CACHE_NONE;
4186 break;
4187 case I915_CACHING_CACHED:
4188
4189
4190
4191
4192
4193
4194 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
4195 return -ENODEV;
4196
4197 level = I915_CACHE_LLC;
4198 break;
4199 case I915_CACHING_DISPLAY:
4200 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
4201 break;
4202 default:
4203 return -EINVAL;
4204 }
4205
4206 obj = i915_gem_object_lookup(file, args->handle);
4207 if (!obj)
4208 return -ENOENT;
4209
4210
4211
4212
4213
4214 if (i915_gem_object_is_proxy(obj)) {
4215 ret = -ENXIO;
4216 goto out;
4217 }
4218
4219 if (obj->cache_level == level)
4220 goto out;
4221
4222 ret = i915_gem_object_wait(obj,
4223 I915_WAIT_INTERRUPTIBLE,
4224 MAX_SCHEDULE_TIMEOUT,
4225 to_rps_client(file));
4226 if (ret)
4227 goto out;
4228
4229 ret = i915_mutex_lock_interruptible(dev);
4230 if (ret)
4231 goto out;
4232
4233 ret = i915_gem_object_set_cache_level(obj, level);
4234 mutex_unlock(&dev->struct_mutex);
4235
4236out:
4237 i915_gem_object_put(obj);
4238 return ret;
4239}
4240
4241
4242
4243
4244
4245
4246
4247struct i915_vma *
4248i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4249 u32 alignment,
4250 const struct i915_ggtt_view *view,
4251 unsigned int flags)
4252{
4253 struct i915_vma *vma;
4254 int ret;
4255
4256 lockdep_assert_held(&obj->base.dev->struct_mutex);
4257
4258
4259
4260
4261 obj->pin_global++;
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272 ret = i915_gem_object_set_cache_level(obj,
4273 HAS_WT(to_i915(obj->base.dev)) ?
4274 I915_CACHE_WT : I915_CACHE_NONE);
4275 if (ret) {
4276 vma = ERR_PTR(ret);
4277 goto err_unpin_global;
4278 }
4279
4280
4281
4282
4283
4284
4285
4286
4287 vma = ERR_PTR(-ENOSPC);
4288 if ((flags & PIN_MAPPABLE) == 0 &&
4289 (!view || view->type == I915_GGTT_VIEW_NORMAL))
4290 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
4291 flags |
4292 PIN_MAPPABLE |
4293 PIN_NONBLOCK);
4294 if (IS_ERR(vma))
4295 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
4296 if (IS_ERR(vma))
4297 goto err_unpin_global;
4298
4299 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
4300
4301 __i915_gem_object_flush_for_display(obj);
4302
4303
4304
4305
4306 obj->read_domains |= I915_GEM_DOMAIN_GTT;
4307
4308 return vma;
4309
4310err_unpin_global:
4311 obj->pin_global--;
4312 return vma;
4313}
4314
4315void
4316i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
4317{
4318 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
4319
4320 if (WARN_ON(vma->obj->pin_global == 0))
4321 return;
4322
4323 if (--vma->obj->pin_global == 0)
4324 vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
4325
4326
4327 i915_gem_object_bump_inactive_ggtt(vma->obj);
4328
4329 i915_vma_unpin(vma);
4330}
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340int
4341i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4342{
4343 int ret;
4344
4345 lockdep_assert_held(&obj->base.dev->struct_mutex);
4346
4347 ret = i915_gem_object_wait(obj,
4348 I915_WAIT_INTERRUPTIBLE |
4349 I915_WAIT_LOCKED |
4350 (write ? I915_WAIT_ALL : 0),
4351 MAX_SCHEDULE_TIMEOUT,
4352 NULL);
4353 if (ret)
4354 return ret;
4355
4356 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
4357
4358
4359 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4360 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
4361 obj->read_domains |= I915_GEM_DOMAIN_CPU;
4362 }
4363
4364
4365
4366
4367 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
4368
4369
4370
4371
4372 if (write)
4373 __start_cpu_write(obj);
4374
4375 return 0;
4376}
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388static int
4389i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4390{
4391 struct drm_i915_private *dev_priv = to_i915(dev);
4392 struct drm_i915_file_private *file_priv = file->driver_priv;
4393 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4394 struct i915_request *request, *target = NULL;
4395 long ret;
4396
4397
4398 if (i915_terminally_wedged(&dev_priv->gpu_error))
4399 return -EIO;
4400
4401 spin_lock(&file_priv->mm.lock);
4402 list_for_each_entry(request, &file_priv->mm.request_list, client_link) {
4403 if (time_after_eq(request->emitted_jiffies, recent_enough))
4404 break;
4405
4406 if (target) {
4407 list_del(&target->client_link);
4408 target->file_priv = NULL;
4409 }
4410
4411 target = request;
4412 }
4413 if (target)
4414 i915_request_get(target);
4415 spin_unlock(&file_priv->mm.lock);
4416
4417 if (target == NULL)
4418 return 0;
4419
4420 ret = i915_request_wait(target,
4421 I915_WAIT_INTERRUPTIBLE,
4422 MAX_SCHEDULE_TIMEOUT);
4423 i915_request_put(target);
4424
4425 return ret < 0 ? ret : 0;
4426}
4427
4428struct i915_vma *
4429i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4430 const struct i915_ggtt_view *view,
4431 u64 size,
4432 u64 alignment,
4433 u64 flags)
4434{
4435 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
4436 struct i915_address_space *vm = &dev_priv->ggtt.vm;
4437 struct i915_vma *vma;
4438 int ret;
4439
4440 lockdep_assert_held(&obj->base.dev->struct_mutex);
4441
4442 if (flags & PIN_MAPPABLE &&
4443 (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
4444
4445
4446
4447
4448
4449
4450
4451 if (obj->base.size > dev_priv->ggtt.mappable_end)
4452 return ERR_PTR(-E2BIG);
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469 if (flags & PIN_NONBLOCK &&
4470 obj->base.size > dev_priv->ggtt.mappable_end / 2)
4471 return ERR_PTR(-ENOSPC);
4472 }
4473
4474 vma = i915_vma_instance(obj, vm, view);
4475 if (unlikely(IS_ERR(vma)))
4476 return vma;
4477
4478 if (i915_vma_misplaced(vma, size, alignment, flags)) {
4479 if (flags & PIN_NONBLOCK) {
4480 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
4481 return ERR_PTR(-ENOSPC);
4482
4483 if (flags & PIN_MAPPABLE &&
4484 vma->fence_size > dev_priv->ggtt.mappable_end / 2)
4485 return ERR_PTR(-ENOSPC);
4486 }
4487
4488 WARN(i915_vma_is_pinned(vma),
4489 "bo is already pinned in ggtt with incorrect alignment:"
4490 " offset=%08x, req.alignment=%llx,"
4491 " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
4492 i915_ggtt_offset(vma), alignment,
4493 !!(flags & PIN_MAPPABLE),
4494 i915_vma_is_map_and_fenceable(vma));
4495 ret = i915_vma_unbind(vma);
4496 if (ret)
4497 return ERR_PTR(ret);
4498 }
4499
4500 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
4501 if (ret)
4502 return ERR_PTR(ret);
4503
4504 return vma;
4505}
4506
4507static __always_inline unsigned int __busy_read_flag(unsigned int id)
4508{
4509
4510
4511
4512
4513
4514
4515
4516 BUILD_BUG_ON(I915_NUM_ENGINES > 16);
4517 return 0x10000 << id;
4518}
4519
4520static __always_inline unsigned int __busy_write_id(unsigned int id)
4521{
4522
4523
4524
4525
4526
4527
4528
4529
4530 return id | __busy_read_flag(id);
4531}
4532
4533static __always_inline unsigned int
4534__busy_set_if_active(const struct dma_fence *fence,
4535 unsigned int (*flag)(unsigned int id))
4536{
4537 struct i915_request *rq;
4538
4539
4540
4541
4542
4543
4544
4545
4546 if (!dma_fence_is_i915(fence))
4547 return 0;
4548
4549
4550 rq = container_of(fence, struct i915_request, fence);
4551 if (i915_request_completed(rq))
4552 return 0;
4553
4554 return flag(rq->engine->uabi_id);
4555}
4556
4557static __always_inline unsigned int
4558busy_check_reader(const struct dma_fence *fence)
4559{
4560 return __busy_set_if_active(fence, __busy_read_flag);
4561}
4562
4563static __always_inline unsigned int
4564busy_check_writer(const struct dma_fence *fence)
4565{
4566 if (!fence)
4567 return 0;
4568
4569 return __busy_set_if_active(fence, __busy_write_id);
4570}
4571
4572int
4573i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4574 struct drm_file *file)
4575{
4576 struct drm_i915_gem_busy *args = data;
4577 struct drm_i915_gem_object *obj;
4578 struct reservation_object_list *list;
4579 unsigned int seq;
4580 int err;
4581
4582 err = -ENOENT;
4583 rcu_read_lock();
4584 obj = i915_gem_object_lookup_rcu(file, args->handle);
4585 if (!obj)
4586 goto out;
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604retry:
4605 seq = raw_read_seqcount(&obj->resv->seq);
4606
4607
4608 args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
4609
4610
4611 list = rcu_dereference(obj->resv->fence);
4612 if (list) {
4613 unsigned int shared_count = list->shared_count, i;
4614
4615 for (i = 0; i < shared_count; ++i) {
4616 struct dma_fence *fence =
4617 rcu_dereference(list->shared[i]);
4618
4619 args->busy |= busy_check_reader(fence);
4620 }
4621 }
4622
4623 if (args->busy && read_seqcount_retry(&obj->resv->seq, seq))
4624 goto retry;
4625
4626 err = 0;
4627out:
4628 rcu_read_unlock();
4629 return err;
4630}
4631
4632int
4633i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4634 struct drm_file *file_priv)
4635{
4636 return i915_gem_ring_throttle(dev, file_priv);
4637}
4638
4639int
4640i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4641 struct drm_file *file_priv)
4642{
4643 struct drm_i915_private *dev_priv = to_i915(dev);
4644 struct drm_i915_gem_madvise *args = data;
4645 struct drm_i915_gem_object *obj;
4646 int err;
4647
4648 switch (args->madv) {
4649 case I915_MADV_DONTNEED:
4650 case I915_MADV_WILLNEED:
4651 break;
4652 default:
4653 return -EINVAL;
4654 }
4655
4656 obj = i915_gem_object_lookup(file_priv, args->handle);
4657 if (!obj)
4658 return -ENOENT;
4659
4660 err = mutex_lock_interruptible(&obj->mm.lock);
4661 if (err)
4662 goto out;
4663
4664 if (i915_gem_object_has_pages(obj) &&
4665 i915_gem_object_is_tiled(obj) &&
4666 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4667 if (obj->mm.madv == I915_MADV_WILLNEED) {
4668 GEM_BUG_ON(!obj->mm.quirked);
4669 __i915_gem_object_unpin_pages(obj);
4670 obj->mm.quirked = false;
4671 }
4672 if (args->madv == I915_MADV_WILLNEED) {
4673 GEM_BUG_ON(obj->mm.quirked);
4674 __i915_gem_object_pin_pages(obj);
4675 obj->mm.quirked = true;
4676 }
4677 }
4678
4679 if (obj->mm.madv != __I915_MADV_PURGED)
4680 obj->mm.madv = args->madv;
4681
4682
4683 if (obj->mm.madv == I915_MADV_DONTNEED &&
4684 !i915_gem_object_has_pages(obj))
4685 i915_gem_object_truncate(obj);
4686
4687 args->retained = obj->mm.madv != __I915_MADV_PURGED;
4688 mutex_unlock(&obj->mm.lock);
4689
4690out:
4691 i915_gem_object_put(obj);
4692 return err;
4693}
4694
4695static void
4696frontbuffer_retire(struct i915_gem_active *active, struct i915_request *request)
4697{
4698 struct drm_i915_gem_object *obj =
4699 container_of(active, typeof(*obj), frontbuffer_write);
4700
4701 intel_fb_obj_flush(obj, ORIGIN_CS);
4702}
4703
4704void i915_gem_object_init(struct drm_i915_gem_object *obj,
4705 const struct drm_i915_gem_object_ops *ops)
4706{
4707 mutex_init(&obj->mm.lock);
4708
4709 INIT_LIST_HEAD(&obj->vma_list);
4710 INIT_LIST_HEAD(&obj->lut_list);
4711 INIT_LIST_HEAD(&obj->batch_pool_link);
4712
4713 obj->ops = ops;
4714
4715 reservation_object_init(&obj->__builtin_resv);
4716 obj->resv = &obj->__builtin_resv;
4717
4718 obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
4719 init_request_active(&obj->frontbuffer_write, frontbuffer_retire);
4720
4721 obj->mm.madv = I915_MADV_WILLNEED;
4722 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
4723 mutex_init(&obj->mm.get_page.lock);
4724
4725 i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4726}
4727
4728static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4729 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
4730 I915_GEM_OBJECT_IS_SHRINKABLE,
4731
4732 .get_pages = i915_gem_object_get_pages_gtt,
4733 .put_pages = i915_gem_object_put_pages_gtt,
4734
4735 .pwrite = i915_gem_object_pwrite_gtt,
4736};
4737
4738static int i915_gem_object_create_shmem(struct drm_device *dev,
4739 struct drm_gem_object *obj,
4740 size_t size)
4741{
4742 struct drm_i915_private *i915 = to_i915(dev);
4743 unsigned long flags = VM_NORESERVE;
4744 struct file *filp;
4745
4746 drm_gem_private_object_init(dev, obj, size);
4747
4748 if (i915->mm.gemfs)
4749 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
4750 flags);
4751 else
4752 filp = shmem_file_setup("i915", size, flags);
4753
4754 if (IS_ERR(filp))
4755 return PTR_ERR(filp);
4756
4757 obj->filp = filp;
4758
4759 return 0;
4760}
4761
4762struct drm_i915_gem_object *
4763i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
4764{
4765 struct drm_i915_gem_object *obj;
4766 struct address_space *mapping;
4767 unsigned int cache_level;
4768 gfp_t mask;
4769 int ret;
4770
4771
4772
4773
4774
4775
4776 if (size >> PAGE_SHIFT > INT_MAX)
4777 return ERR_PTR(-E2BIG);
4778
4779 if (overflows_type(size, obj->base.size))
4780 return ERR_PTR(-E2BIG);
4781
4782 obj = i915_gem_object_alloc(dev_priv);
4783 if (obj == NULL)
4784 return ERR_PTR(-ENOMEM);
4785
4786 ret = i915_gem_object_create_shmem(&dev_priv->drm, &obj->base, size);
4787 if (ret)
4788 goto fail;
4789
4790 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4791 if (IS_I965GM(dev_priv) || IS_I965G(dev_priv)) {
4792
4793 mask &= ~__GFP_HIGHMEM;
4794 mask |= __GFP_DMA32;
4795 }
4796
4797 mapping = obj->base.filp->f_mapping;
4798 mapping_set_gfp_mask(mapping, mask);
4799 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
4800
4801 i915_gem_object_init(obj, &i915_gem_object_ops);
4802
4803 obj->write_domain = I915_GEM_DOMAIN_CPU;
4804 obj->read_domains = I915_GEM_DOMAIN_CPU;
4805
4806 if (HAS_LLC(dev_priv))
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819 cache_level = I915_CACHE_LLC;
4820 else
4821 cache_level = I915_CACHE_NONE;
4822
4823 i915_gem_object_set_cache_coherency(obj, cache_level);
4824
4825 trace_i915_gem_object_create(obj);
4826
4827 return obj;
4828
4829fail:
4830 i915_gem_object_free(obj);
4831 return ERR_PTR(ret);
4832}
4833
4834static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4835{
4836
4837
4838
4839
4840
4841
4842 if (obj->mm.madv != I915_MADV_WILLNEED)
4843 return false;
4844
4845 if (obj->base.filp == NULL)
4846 return true;
4847
4848
4849
4850
4851
4852
4853
4854
4855 return atomic_long_read(&obj->base.filp->f_count) == 1;
4856}
4857
4858static void __i915_gem_free_objects(struct drm_i915_private *i915,
4859 struct llist_node *freed)
4860{
4861 struct drm_i915_gem_object *obj, *on;
4862
4863 intel_runtime_pm_get(i915);
4864 llist_for_each_entry_safe(obj, on, freed, freed) {
4865 struct i915_vma *vma, *vn;
4866
4867 trace_i915_gem_object_destroy(obj);
4868
4869 mutex_lock(&i915->drm.struct_mutex);
4870
4871 GEM_BUG_ON(i915_gem_object_is_active(obj));
4872 list_for_each_entry_safe(vma, vn,
4873 &obj->vma_list, obj_link) {
4874 GEM_BUG_ON(i915_vma_is_active(vma));
4875 vma->flags &= ~I915_VMA_PIN_MASK;
4876 i915_vma_destroy(vma);
4877 }
4878 GEM_BUG_ON(!list_empty(&obj->vma_list));
4879 GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree));
4880
4881
4882
4883
4884
4885
4886
4887 if (i915_gem_object_has_pages(obj)) {
4888 spin_lock(&i915->mm.obj_lock);
4889 list_del_init(&obj->mm.link);
4890 spin_unlock(&i915->mm.obj_lock);
4891 }
4892
4893 mutex_unlock(&i915->drm.struct_mutex);
4894
4895 GEM_BUG_ON(obj->bind_count);
4896 GEM_BUG_ON(obj->userfault_count);
4897 GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
4898 GEM_BUG_ON(!list_empty(&obj->lut_list));
4899
4900 if (obj->ops->release)
4901 obj->ops->release(obj);
4902
4903 if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
4904 atomic_set(&obj->mm.pages_pin_count, 0);
4905 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
4906 GEM_BUG_ON(i915_gem_object_has_pages(obj));
4907
4908 if (obj->base.import_attach)
4909 drm_prime_gem_destroy(&obj->base, NULL);
4910
4911 reservation_object_fini(&obj->__builtin_resv);
4912 drm_gem_object_release(&obj->base);
4913 i915_gem_info_remove_obj(i915, obj->base.size);
4914
4915 kfree(obj->bit_17);
4916 i915_gem_object_free(obj);
4917
4918 GEM_BUG_ON(!atomic_read(&i915->mm.free_count));
4919 atomic_dec(&i915->mm.free_count);
4920
4921 if (on)
4922 cond_resched();
4923 }
4924 intel_runtime_pm_put(i915);
4925}
4926
4927static void i915_gem_flush_free_objects(struct drm_i915_private *i915)
4928{
4929 struct llist_node *freed;
4930
4931
4932 freed = NULL;
4933 if (!llist_empty(&i915->mm.free_list)) {
4934
4935 spin_lock(&i915->mm.free_lock);
4936 freed = llist_del_first(&i915->mm.free_list);
4937 spin_unlock(&i915->mm.free_lock);
4938 }
4939 if (unlikely(freed)) {
4940 freed->next = NULL;
4941 __i915_gem_free_objects(i915, freed);
4942 }
4943}
4944
4945static void __i915_gem_free_work(struct work_struct *work)
4946{
4947 struct drm_i915_private *i915 =
4948 container_of(work, struct drm_i915_private, mm.free_work);
4949 struct llist_node *freed;
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960 spin_lock(&i915->mm.free_lock);
4961 while ((freed = llist_del_all(&i915->mm.free_list))) {
4962 spin_unlock(&i915->mm.free_lock);
4963
4964 __i915_gem_free_objects(i915, freed);
4965 if (need_resched())
4966 return;
4967
4968 spin_lock(&i915->mm.free_lock);
4969 }
4970 spin_unlock(&i915->mm.free_lock);
4971}
4972
4973static void __i915_gem_free_object_rcu(struct rcu_head *head)
4974{
4975 struct drm_i915_gem_object *obj =
4976 container_of(head, typeof(*obj), rcu);
4977 struct drm_i915_private *i915 = to_i915(obj->base.dev);
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989 if (llist_add(&obj->freed, &i915->mm.free_list))
4990 queue_work(i915->wq, &i915->mm.free_work);
4991}
4992
4993void i915_gem_free_object(struct drm_gem_object *gem_obj)
4994{
4995 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4996
4997 if (obj->mm.quirked)
4998 __i915_gem_object_unpin_pages(obj);
4999
5000 if (discard_backing_storage(obj))
5001 obj->mm.madv = I915_MADV_DONTNEED;
5002
5003
5004
5005
5006
5007
5008
5009 atomic_inc(&to_i915(obj->base.dev)->mm.free_count);
5010 call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
5011}
5012
5013void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
5014{
5015 lockdep_assert_held(&obj->base.dev->struct_mutex);
5016
5017 if (!i915_gem_object_has_active_reference(obj) &&
5018 i915_gem_object_is_active(obj))
5019 i915_gem_object_set_active_reference(obj);
5020 else
5021 i915_gem_object_put(obj);
5022}
5023
5024void i915_gem_sanitize(struct drm_i915_private *i915)
5025{
5026 int err;
5027
5028 GEM_TRACE("\n");
5029
5030 mutex_lock(&i915->drm.struct_mutex);
5031
5032 intel_runtime_pm_get(i915);
5033 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
5034
5035
5036
5037
5038
5039
5040
5041 if (i915_terminally_wedged(&i915->gpu_error))
5042 i915_gem_unset_wedged(i915);
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052 err = -ENODEV;
5053 if (INTEL_GEN(i915) >= 5 && intel_has_gpu_reset(i915))
5054 err = WARN_ON(intel_gpu_reset(i915, ALL_ENGINES));
5055 if (!err)
5056 intel_engines_sanitize(i915);
5057
5058 intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
5059 intel_runtime_pm_put(i915);
5060
5061 i915_gem_contexts_lost(i915);
5062 mutex_unlock(&i915->drm.struct_mutex);
5063}
5064
5065int i915_gem_suspend(struct drm_i915_private *i915)
5066{
5067 int ret;
5068
5069 GEM_TRACE("\n");
5070
5071 intel_runtime_pm_get(i915);
5072 intel_suspend_gt_powersave(i915);
5073
5074 mutex_lock(&i915->drm.struct_mutex);
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085 if (!i915_terminally_wedged(&i915->gpu_error)) {
5086 ret = i915_gem_switch_to_kernel_context(i915);
5087 if (ret)
5088 goto err_unlock;
5089
5090 ret = i915_gem_wait_for_idle(i915,
5091 I915_WAIT_INTERRUPTIBLE |
5092 I915_WAIT_LOCKED |
5093 I915_WAIT_FOR_IDLE_BOOST,
5094 MAX_SCHEDULE_TIMEOUT);
5095 if (ret && ret != -EIO)
5096 goto err_unlock;
5097
5098 assert_kernel_context_is_current(i915);
5099 }
5100 i915_retire_requests(i915);
5101
5102 mutex_unlock(&i915->drm.struct_mutex);
5103
5104 intel_uc_suspend(i915);
5105
5106 cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
5107 cancel_delayed_work_sync(&i915->gt.retire_work);
5108
5109
5110
5111
5112
5113 drain_delayed_work(&i915->gt.idle_work);
5114
5115
5116
5117
5118
5119 WARN_ON(i915->gt.awake);
5120 if (WARN_ON(!intel_engines_are_idle(i915)))
5121 i915_gem_set_wedged(i915);
5122
5123 intel_runtime_pm_put(i915);
5124 return 0;
5125
5126err_unlock:
5127 mutex_unlock(&i915->drm.struct_mutex);
5128 intel_runtime_pm_put(i915);
5129 return ret;
5130}
5131
5132void i915_gem_suspend_late(struct drm_i915_private *i915)
5133{
5134 struct drm_i915_gem_object *obj;
5135 struct list_head *phases[] = {
5136 &i915->mm.unbound_list,
5137 &i915->mm.bound_list,
5138 NULL
5139 }, **phase;
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161 mutex_lock(&i915->drm.struct_mutex);
5162 for (phase = phases; *phase; phase++) {
5163 list_for_each_entry(obj, *phase, mm.link)
5164 WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
5165 }
5166 mutex_unlock(&i915->drm.struct_mutex);
5167
5168 intel_uc_sanitize(i915);
5169 i915_gem_sanitize(i915);
5170}
5171
5172void i915_gem_resume(struct drm_i915_private *i915)
5173{
5174 GEM_TRACE("\n");
5175
5176 WARN_ON(i915->gt.awake);
5177
5178 mutex_lock(&i915->drm.struct_mutex);
5179 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
5180
5181 i915_gem_restore_gtt_mappings(i915);
5182 i915_gem_restore_fences(i915);
5183
5184
5185
5186
5187
5188
5189 i915->gt.resume(i915);
5190
5191 if (i915_gem_init_hw(i915))
5192 goto err_wedged;
5193
5194 intel_uc_resume(i915);
5195
5196
5197 if (i915_gem_switch_to_kernel_context(i915))
5198 goto err_wedged;
5199
5200out_unlock:
5201 intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
5202 mutex_unlock(&i915->drm.struct_mutex);
5203 return;
5204
5205err_wedged:
5206 if (!i915_terminally_wedged(&i915->gpu_error)) {
5207 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
5208 i915_gem_set_wedged(i915);
5209 }
5210 goto out_unlock;
5211}
5212
5213void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
5214{
5215 if (INTEL_GEN(dev_priv) < 5 ||
5216 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
5217 return;
5218
5219 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
5220 DISP_TILE_SURFACE_SWIZZLING);
5221
5222 if (IS_GEN5(dev_priv))
5223 return;
5224
5225 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
5226 if (IS_GEN6(dev_priv))
5227 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
5228 else if (IS_GEN7(dev_priv))
5229 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
5230 else if (IS_GEN8(dev_priv))
5231 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
5232 else
5233 BUG();
5234}
5235
5236static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
5237{
5238 I915_WRITE(RING_CTL(base), 0);
5239 I915_WRITE(RING_HEAD(base), 0);
5240 I915_WRITE(RING_TAIL(base), 0);
5241 I915_WRITE(RING_START(base), 0);
5242}
5243
5244static void init_unused_rings(struct drm_i915_private *dev_priv)
5245{
5246 if (IS_I830(dev_priv)) {
5247 init_unused_ring(dev_priv, PRB1_BASE);
5248 init_unused_ring(dev_priv, SRB0_BASE);
5249 init_unused_ring(dev_priv, SRB1_BASE);
5250 init_unused_ring(dev_priv, SRB2_BASE);
5251 init_unused_ring(dev_priv, SRB3_BASE);
5252 } else if (IS_GEN2(dev_priv)) {
5253 init_unused_ring(dev_priv, SRB0_BASE);
5254 init_unused_ring(dev_priv, SRB1_BASE);
5255 } else if (IS_GEN3(dev_priv)) {
5256 init_unused_ring(dev_priv, PRB1_BASE);
5257 init_unused_ring(dev_priv, PRB2_BASE);
5258 }
5259}
5260
5261static int __i915_gem_restart_engines(void *data)
5262{
5263 struct drm_i915_private *i915 = data;
5264 struct intel_engine_cs *engine;
5265 enum intel_engine_id id;
5266 int err;
5267
5268 for_each_engine(engine, i915, id) {
5269 err = engine->init_hw(engine);
5270 if (err) {
5271 DRM_ERROR("Failed to restart %s (%d)\n",
5272 engine->name, err);
5273 return err;
5274 }
5275 }
5276
5277 return 0;
5278}
5279
5280int i915_gem_init_hw(struct drm_i915_private *dev_priv)
5281{
5282 int ret;
5283
5284 dev_priv->gt.last_init_time = ktime_get();
5285
5286
5287 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5288
5289 if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
5290 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
5291
5292 if (IS_HASWELL(dev_priv))
5293 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
5294 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
5295
5296 if (HAS_PCH_NOP(dev_priv)) {
5297 if (IS_IVYBRIDGE(dev_priv)) {
5298 u32 temp = I915_READ(GEN7_MSG_CTL);
5299 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
5300 I915_WRITE(GEN7_MSG_CTL, temp);
5301 } else if (INTEL_GEN(dev_priv) >= 7) {
5302 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
5303 temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
5304 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
5305 }
5306 }
5307
5308 intel_gt_apply_workarounds(dev_priv);
5309
5310 i915_gem_init_swizzling(dev_priv);
5311
5312
5313
5314
5315
5316
5317
5318 init_unused_rings(dev_priv);
5319
5320 BUG_ON(!dev_priv->kernel_context);
5321 if (i915_terminally_wedged(&dev_priv->gpu_error)) {
5322 ret = -EIO;
5323 goto out;
5324 }
5325
5326 ret = i915_ppgtt_init_hw(dev_priv);
5327 if (ret) {
5328 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
5329 goto out;
5330 }
5331
5332 ret = intel_wopcm_init_hw(&dev_priv->wopcm);
5333 if (ret) {
5334 DRM_ERROR("Enabling WOPCM failed (%d)\n", ret);
5335 goto out;
5336 }
5337
5338
5339 ret = intel_uc_init_hw(dev_priv);
5340 if (ret) {
5341 DRM_ERROR("Enabling uc failed (%d)\n", ret);
5342 goto out;
5343 }
5344
5345 intel_mocs_init_l3cc_table(dev_priv);
5346
5347
5348 ret = __i915_gem_restart_engines(dev_priv);
5349 if (ret)
5350 goto cleanup_uc;
5351
5352 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5353
5354 return 0;
5355
5356cleanup_uc:
5357 intel_uc_fini_hw(dev_priv);
5358out:
5359 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5360
5361 return ret;
5362}
5363
5364static int __intel_engines_record_defaults(struct drm_i915_private *i915)
5365{
5366 struct i915_gem_context *ctx;
5367 struct intel_engine_cs *engine;
5368 enum intel_engine_id id;
5369 int err;
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380 ctx = i915_gem_context_create_kernel(i915, 0);
5381 if (IS_ERR(ctx))
5382 return PTR_ERR(ctx);
5383
5384 for_each_engine(engine, i915, id) {
5385 struct i915_request *rq;
5386
5387 rq = i915_request_alloc(engine, ctx);
5388 if (IS_ERR(rq)) {
5389 err = PTR_ERR(rq);
5390 goto out_ctx;
5391 }
5392
5393 err = 0;
5394 if (engine->init_context)
5395 err = engine->init_context(rq);
5396
5397 i915_request_add(rq);
5398 if (err)
5399 goto err_active;
5400 }
5401
5402 err = i915_gem_switch_to_kernel_context(i915);
5403 if (err)
5404 goto err_active;
5405
5406 if (i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED, HZ / 5)) {
5407 i915_gem_set_wedged(i915);
5408 err = -EIO;
5409 goto err_active;
5410 }
5411
5412 assert_kernel_context_is_current(i915);
5413
5414
5415
5416
5417
5418
5419
5420 __i915_gem_park(i915);
5421
5422 for_each_engine(engine, i915, id) {
5423 struct i915_vma *state;
5424 void *vaddr;
5425
5426 GEM_BUG_ON(to_intel_context(ctx, engine)->pin_count);
5427
5428 state = to_intel_context(ctx, engine)->state;
5429 if (!state)
5430 continue;
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440 err = i915_vma_unbind(state);
5441 if (err)
5442 goto err_active;
5443
5444 err = i915_gem_object_set_to_cpu_domain(state->obj, false);
5445 if (err)
5446 goto err_active;
5447
5448 engine->default_state = i915_gem_object_get(state->obj);
5449
5450
5451 vaddr = i915_gem_object_pin_map(engine->default_state,
5452 I915_MAP_FORCE_WB);
5453 if (IS_ERR(vaddr)) {
5454 err = PTR_ERR(vaddr);
5455 goto err_active;
5456 }
5457
5458 i915_gem_object_unpin_map(engine->default_state);
5459 }
5460
5461 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
5462 unsigned int found = intel_engines_has_context_isolation(i915);
5463
5464
5465
5466
5467
5468 for_each_engine(engine, i915, id) {
5469 unsigned int bit = BIT(engine->uabi_class);
5470 unsigned int expected = engine->default_state ? bit : 0;
5471
5472 if ((found & bit) != expected) {
5473 DRM_ERROR("mismatching default context state for class %d on engine %s\n",
5474 engine->uabi_class, engine->name);
5475 }
5476 }
5477 }
5478
5479out_ctx:
5480 i915_gem_context_set_closed(ctx);
5481 i915_gem_context_put(ctx);
5482 return err;
5483
5484err_active:
5485
5486
5487
5488
5489
5490
5491 if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
5492 goto out_ctx;
5493
5494 if (WARN_ON(i915_gem_wait_for_idle(i915,
5495 I915_WAIT_LOCKED,
5496 MAX_SCHEDULE_TIMEOUT)))
5497 goto out_ctx;
5498
5499 i915_gem_contexts_lost(i915);
5500 goto out_ctx;
5501}
5502
5503static int
5504i915_gem_init_scratch(struct drm_i915_private *i915, unsigned int size)
5505{
5506 struct drm_i915_gem_object *obj;
5507 struct i915_vma *vma;
5508 int ret;
5509
5510 obj = i915_gem_object_create_stolen(i915, size);
5511 if (!obj)
5512 obj = i915_gem_object_create_internal(i915, size);
5513 if (IS_ERR(obj)) {
5514 DRM_ERROR("Failed to allocate scratch page\n");
5515 return PTR_ERR(obj);
5516 }
5517
5518 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
5519 if (IS_ERR(vma)) {
5520 ret = PTR_ERR(vma);
5521 goto err_unref;
5522 }
5523
5524 ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
5525 if (ret)
5526 goto err_unref;
5527
5528 i915->gt.scratch = vma;
5529 return 0;
5530
5531err_unref:
5532 i915_gem_object_put(obj);
5533 return ret;
5534}
5535
5536static void i915_gem_fini_scratch(struct drm_i915_private *i915)
5537{
5538 i915_vma_unpin_and_release(&i915->gt.scratch, 0);
5539}
5540
5541int i915_gem_init(struct drm_i915_private *dev_priv)
5542{
5543 int ret;
5544
5545
5546 if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
5547 mkwrite_device_info(dev_priv)->page_sizes =
5548 I915_GTT_PAGE_SIZE_4K;
5549
5550 dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
5551
5552 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
5553 dev_priv->gt.resume = intel_lr_context_resume;
5554 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
5555 } else {
5556 dev_priv->gt.resume = intel_legacy_submission_resume;
5557 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
5558 }
5559
5560 ret = i915_gem_init_userptr(dev_priv);
5561 if (ret)
5562 return ret;
5563
5564 ret = intel_uc_init_misc(dev_priv);
5565 if (ret)
5566 return ret;
5567
5568 ret = intel_wopcm_init(&dev_priv->wopcm);
5569 if (ret)
5570 goto err_uc_misc;
5571
5572
5573
5574
5575
5576
5577
5578 mutex_lock(&dev_priv->drm.struct_mutex);
5579 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5580
5581 ret = i915_gem_init_ggtt(dev_priv);
5582 if (ret) {
5583 GEM_BUG_ON(ret == -EIO);
5584 goto err_unlock;
5585 }
5586
5587 ret = i915_gem_init_scratch(dev_priv,
5588 IS_GEN2(dev_priv) ? SZ_256K : PAGE_SIZE);
5589 if (ret) {
5590 GEM_BUG_ON(ret == -EIO);
5591 goto err_ggtt;
5592 }
5593
5594 ret = i915_gem_contexts_init(dev_priv);
5595 if (ret) {
5596 GEM_BUG_ON(ret == -EIO);
5597 goto err_scratch;
5598 }
5599
5600 ret = intel_engines_init(dev_priv);
5601 if (ret) {
5602 GEM_BUG_ON(ret == -EIO);
5603 goto err_context;
5604 }
5605
5606 intel_init_gt_powersave(dev_priv);
5607
5608 ret = intel_uc_init(dev_priv);
5609 if (ret)
5610 goto err_pm;
5611
5612 ret = i915_gem_init_hw(dev_priv);
5613 if (ret)
5614 goto err_uc_init;
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625 intel_init_clock_gating(dev_priv);
5626
5627 ret = __intel_engines_record_defaults(dev_priv);
5628 if (ret)
5629 goto err_init_hw;
5630
5631 if (i915_inject_load_failure()) {
5632 ret = -ENODEV;
5633 goto err_init_hw;
5634 }
5635
5636 if (i915_inject_load_failure()) {
5637 ret = -EIO;
5638 goto err_init_hw;
5639 }
5640
5641 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5642 mutex_unlock(&dev_priv->drm.struct_mutex);
5643
5644 return 0;
5645
5646
5647
5648
5649
5650
5651
5652err_init_hw:
5653 mutex_unlock(&dev_priv->drm.struct_mutex);
5654
5655 WARN_ON(i915_gem_suspend(dev_priv));
5656 i915_gem_suspend_late(dev_priv);
5657
5658 i915_gem_drain_workqueue(dev_priv);
5659
5660 mutex_lock(&dev_priv->drm.struct_mutex);
5661 intel_uc_fini_hw(dev_priv);
5662err_uc_init:
5663 intel_uc_fini(dev_priv);
5664err_pm:
5665 if (ret != -EIO) {
5666 intel_cleanup_gt_powersave(dev_priv);
5667 i915_gem_cleanup_engines(dev_priv);
5668 }
5669err_context:
5670 if (ret != -EIO)
5671 i915_gem_contexts_fini(dev_priv);
5672err_scratch:
5673 i915_gem_fini_scratch(dev_priv);
5674err_ggtt:
5675err_unlock:
5676 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5677 mutex_unlock(&dev_priv->drm.struct_mutex);
5678
5679err_uc_misc:
5680 intel_uc_fini_misc(dev_priv);
5681
5682 if (ret != -EIO)
5683 i915_gem_cleanup_userptr(dev_priv);
5684
5685 if (ret == -EIO) {
5686 mutex_lock(&dev_priv->drm.struct_mutex);
5687
5688
5689
5690
5691
5692
5693 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
5694 i915_load_error(dev_priv,
5695 "Failed to initialize GPU, declaring it wedged!\n");
5696 i915_gem_set_wedged(dev_priv);
5697 }
5698
5699
5700 ret = i915_ggtt_enable_hw(dev_priv);
5701 i915_gem_restore_gtt_mappings(dev_priv);
5702 i915_gem_restore_fences(dev_priv);
5703 intel_init_clock_gating(dev_priv);
5704
5705 mutex_unlock(&dev_priv->drm.struct_mutex);
5706 }
5707
5708 i915_gem_drain_freed_objects(dev_priv);
5709 return ret;
5710}
5711
5712void i915_gem_fini(struct drm_i915_private *dev_priv)
5713{
5714 i915_gem_suspend_late(dev_priv);
5715 intel_disable_gt_powersave(dev_priv);
5716
5717
5718 i915_gem_drain_workqueue(dev_priv);
5719
5720 mutex_lock(&dev_priv->drm.struct_mutex);
5721 intel_uc_fini_hw(dev_priv);
5722 intel_uc_fini(dev_priv);
5723 i915_gem_cleanup_engines(dev_priv);
5724 i915_gem_contexts_fini(dev_priv);
5725 i915_gem_fini_scratch(dev_priv);
5726 mutex_unlock(&dev_priv->drm.struct_mutex);
5727
5728 intel_wa_list_free(&dev_priv->gt_wa_list);
5729
5730 intel_cleanup_gt_powersave(dev_priv);
5731
5732 intel_uc_fini_misc(dev_priv);
5733 i915_gem_cleanup_userptr(dev_priv);
5734
5735 i915_gem_drain_freed_objects(dev_priv);
5736
5737 WARN_ON(!list_empty(&dev_priv->contexts.list));
5738}
5739
5740void i915_gem_init_mmio(struct drm_i915_private *i915)
5741{
5742 i915_gem_sanitize(i915);
5743}
5744
5745void
5746i915_gem_cleanup_engines(struct drm_i915_private *dev_priv)
5747{
5748 struct intel_engine_cs *engine;
5749 enum intel_engine_id id;
5750
5751 for_each_engine(engine, dev_priv, id)
5752 dev_priv->gt.cleanup_engine(engine);
5753}
5754
5755void
5756i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
5757{
5758 int i;
5759
5760 if (INTEL_GEN(dev_priv) >= 7 && !IS_VALLEYVIEW(dev_priv) &&
5761 !IS_CHERRYVIEW(dev_priv))
5762 dev_priv->num_fence_regs = 32;
5763 else if (INTEL_GEN(dev_priv) >= 4 ||
5764 IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
5765 IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
5766 dev_priv->num_fence_regs = 16;
5767 else
5768 dev_priv->num_fence_regs = 8;
5769
5770 if (intel_vgpu_active(dev_priv))
5771 dev_priv->num_fence_regs =
5772 I915_READ(vgtif_reg(avail_rs.fence_num));
5773
5774
5775 for (i = 0; i < dev_priv->num_fence_regs; i++) {
5776 struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
5777
5778 fence->i915 = dev_priv;
5779 fence->id = i;
5780 list_add_tail(&fence->link, &dev_priv->mm.fence_list);
5781 }
5782 i915_gem_restore_fences(dev_priv);
5783
5784 i915_gem_detect_bit_6_swizzle(dev_priv);
5785}
5786
5787static void i915_gem_init__mm(struct drm_i915_private *i915)
5788{
5789 spin_lock_init(&i915->mm.object_stat_lock);
5790 spin_lock_init(&i915->mm.obj_lock);
5791 spin_lock_init(&i915->mm.free_lock);
5792
5793 init_llist_head(&i915->mm.free_list);
5794
5795 INIT_LIST_HEAD(&i915->mm.unbound_list);
5796 INIT_LIST_HEAD(&i915->mm.bound_list);
5797 INIT_LIST_HEAD(&i915->mm.fence_list);
5798 INIT_LIST_HEAD(&i915->mm.userfault_list);
5799
5800 INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
5801}
5802
5803int i915_gem_init_early(struct drm_i915_private *dev_priv)
5804{
5805 int err = -ENOMEM;
5806
5807 dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
5808 if (!dev_priv->objects)
5809 goto err_out;
5810
5811 dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
5812 if (!dev_priv->vmas)
5813 goto err_objects;
5814
5815 dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0);
5816 if (!dev_priv->luts)
5817 goto err_vmas;
5818
5819 dev_priv->requests = KMEM_CACHE(i915_request,
5820 SLAB_HWCACHE_ALIGN |
5821 SLAB_RECLAIM_ACCOUNT |
5822 SLAB_TYPESAFE_BY_RCU);
5823 if (!dev_priv->requests)
5824 goto err_luts;
5825
5826 dev_priv->dependencies = KMEM_CACHE(i915_dependency,
5827 SLAB_HWCACHE_ALIGN |
5828 SLAB_RECLAIM_ACCOUNT);
5829 if (!dev_priv->dependencies)
5830 goto err_requests;
5831
5832 dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
5833 if (!dev_priv->priorities)
5834 goto err_dependencies;
5835
5836 INIT_LIST_HEAD(&dev_priv->gt.timelines);
5837 INIT_LIST_HEAD(&dev_priv->gt.active_rings);
5838 INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
5839
5840 i915_gem_init__mm(dev_priv);
5841
5842 INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
5843 i915_gem_retire_work_handler);
5844 INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
5845 i915_gem_idle_work_handler);
5846 init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
5847 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5848
5849 atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
5850
5851 spin_lock_init(&dev_priv->fb_tracking.lock);
5852
5853 err = i915_gemfs_init(dev_priv);
5854 if (err)
5855 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);
5856
5857 return 0;
5858
5859err_dependencies:
5860 kmem_cache_destroy(dev_priv->dependencies);
5861err_requests:
5862 kmem_cache_destroy(dev_priv->requests);
5863err_luts:
5864 kmem_cache_destroy(dev_priv->luts);
5865err_vmas:
5866 kmem_cache_destroy(dev_priv->vmas);
5867err_objects:
5868 kmem_cache_destroy(dev_priv->objects);
5869err_out:
5870 return err;
5871}
5872
5873void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
5874{
5875 i915_gem_drain_freed_objects(dev_priv);
5876 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
5877 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
5878 WARN_ON(dev_priv->mm.object_count);
5879 WARN_ON(!list_empty(&dev_priv->gt.timelines));
5880
5881 kmem_cache_destroy(dev_priv->priorities);
5882 kmem_cache_destroy(dev_priv->dependencies);
5883 kmem_cache_destroy(dev_priv->requests);
5884 kmem_cache_destroy(dev_priv->luts);
5885 kmem_cache_destroy(dev_priv->vmas);
5886 kmem_cache_destroy(dev_priv->objects);
5887
5888
5889 rcu_barrier();
5890
5891 i915_gemfs_fini(dev_priv);
5892}
5893
5894int i915_gem_freeze(struct drm_i915_private *dev_priv)
5895{
5896
5897
5898
5899 i915_gem_shrink_all(dev_priv);
5900
5901 return 0;
5902}
5903
5904int i915_gem_freeze_late(struct drm_i915_private *i915)
5905{
5906 struct drm_i915_gem_object *obj;
5907 struct list_head *phases[] = {
5908 &i915->mm.unbound_list,
5909 &i915->mm.bound_list,
5910 NULL
5911 }, **phase;
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928 i915_gem_shrink(i915, -1UL, NULL, I915_SHRINK_UNBOUND);
5929 i915_gem_drain_freed_objects(i915);
5930
5931 mutex_lock(&i915->drm.struct_mutex);
5932 for (phase = phases; *phase; phase++) {
5933 list_for_each_entry(obj, *phase, mm.link)
5934 WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true));
5935 }
5936 mutex_unlock(&i915->drm.struct_mutex);
5937
5938 return 0;
5939}
5940
5941void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5942{
5943 struct drm_i915_file_private *file_priv = file->driver_priv;
5944 struct i915_request *request;
5945
5946
5947
5948
5949
5950 spin_lock(&file_priv->mm.lock);
5951 list_for_each_entry(request, &file_priv->mm.request_list, client_link)
5952 request->file_priv = NULL;
5953 spin_unlock(&file_priv->mm.lock);
5954}
5955
5956int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
5957{
5958 struct drm_i915_file_private *file_priv;
5959 int ret;
5960
5961 DRM_DEBUG("\n");
5962
5963 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5964 if (!file_priv)
5965 return -ENOMEM;
5966
5967 file->driver_priv = file_priv;
5968 file_priv->dev_priv = i915;
5969 file_priv->file = file;
5970
5971 spin_lock_init(&file_priv->mm.lock);
5972 INIT_LIST_HEAD(&file_priv->mm.request_list);
5973
5974 file_priv->bsd_engine = -1;
5975 file_priv->hang_timestamp = jiffies;
5976
5977 ret = i915_gem_context_open(i915, file);
5978 if (ret)
5979 kfree(file_priv);
5980
5981 return ret;
5982}
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993void i915_gem_track_fb(struct drm_i915_gem_object *old,
5994 struct drm_i915_gem_object *new,
5995 unsigned frontbuffer_bits)
5996{
5997
5998
5999
6000
6001
6002
6003 BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
6004 sizeof(atomic_t) * BITS_PER_BYTE);
6005
6006 if (old) {
6007 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
6008 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
6009 }
6010
6011 if (new) {
6012 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
6013 atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
6014 }
6015}
6016
6017
6018struct drm_i915_gem_object *
6019i915_gem_object_create_from_data(struct drm_i915_private *dev_priv,
6020 const void *data, size_t size)
6021{
6022 struct drm_i915_gem_object *obj;
6023 struct file *file;
6024 size_t offset;
6025 int err;
6026
6027 obj = i915_gem_object_create(dev_priv, round_up(size, PAGE_SIZE));
6028 if (IS_ERR(obj))
6029 return obj;
6030
6031 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
6032
6033 file = obj->base.filp;
6034 offset = 0;
6035 do {
6036 unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
6037 struct page *page;
6038 void *pgdata, *vaddr;
6039
6040 err = pagecache_write_begin(file, file->f_mapping,
6041 offset, len, 0,
6042 &page, &pgdata);
6043 if (err < 0)
6044 goto fail;
6045
6046 vaddr = kmap(page);
6047 memcpy(vaddr, data, len);
6048 kunmap(page);
6049
6050 err = pagecache_write_end(file, file->f_mapping,
6051 offset, len, len,
6052 page, pgdata);
6053 if (err < 0)
6054 goto fail;
6055
6056 size -= len;
6057 data += len;
6058 offset += len;
6059 } while (size);
6060
6061 return obj;
6062
6063fail:
6064 i915_gem_object_put(obj);
6065 return ERR_PTR(err);
6066}
6067
6068struct scatterlist *
6069i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
6070 unsigned int n,
6071 unsigned int *offset)
6072{
6073 struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
6074 struct scatterlist *sg;
6075 unsigned int idx, count;
6076
6077 might_sleep();
6078 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
6079 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090 if (n < READ_ONCE(iter->sg_idx))
6091 goto lookup;
6092
6093 mutex_lock(&iter->lock);
6094
6095
6096
6097
6098
6099
6100 sg = iter->sg_pos;
6101 idx = iter->sg_idx;
6102 count = __sg_page_count(sg);
6103
6104 while (idx + count <= n) {
6105 void *entry;
6106 unsigned long i;
6107 int ret;
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117 ret = radix_tree_insert(&iter->radix, idx, sg);
6118 if (ret && ret != -EEXIST)
6119 goto scan;
6120
6121 entry = xa_mk_value(idx);
6122 for (i = 1; i < count; i++) {
6123 ret = radix_tree_insert(&iter->radix, idx + i, entry);
6124 if (ret && ret != -EEXIST)
6125 goto scan;
6126 }
6127
6128 idx += count;
6129 sg = ____sg_next(sg);
6130 count = __sg_page_count(sg);
6131 }
6132
6133scan:
6134 iter->sg_pos = sg;
6135 iter->sg_idx = idx;
6136
6137 mutex_unlock(&iter->lock);
6138
6139 if (unlikely(n < idx))
6140 goto lookup;
6141
6142
6143
6144
6145 while (idx + count <= n) {
6146 idx += count;
6147 sg = ____sg_next(sg);
6148 count = __sg_page_count(sg);
6149 }
6150
6151 *offset = n - idx;
6152 return sg;
6153
6154lookup:
6155 rcu_read_lock();
6156
6157 sg = radix_tree_lookup(&iter->radix, n);
6158 GEM_BUG_ON(!sg);
6159
6160
6161
6162
6163
6164
6165
6166 *offset = 0;
6167 if (unlikely(xa_is_value(sg))) {
6168 unsigned long base = xa_to_value(sg);
6169
6170 sg = radix_tree_lookup(&iter->radix, base);
6171 GEM_BUG_ON(!sg);
6172
6173 *offset = n - base;
6174 }
6175
6176 rcu_read_unlock();
6177
6178 return sg;
6179}
6180
6181struct page *
6182i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
6183{
6184 struct scatterlist *sg;
6185 unsigned int offset;
6186
6187 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
6188
6189 sg = i915_gem_object_get_sg(obj, n, &offset);
6190 return nth_page(sg_page(sg), offset);
6191}
6192
6193
6194struct page *
6195i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
6196 unsigned int n)
6197{
6198 struct page *page;
6199
6200 page = i915_gem_object_get_page(obj, n);
6201 if (!obj->mm.dirty)
6202 set_page_dirty(page);
6203
6204 return page;
6205}
6206
6207dma_addr_t
6208i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
6209 unsigned long n)
6210{
6211 struct scatterlist *sg;
6212 unsigned int offset;
6213
6214 sg = i915_gem_object_get_sg(obj, n, &offset);
6215 return sg_dma_address(sg) + (offset << PAGE_SHIFT);
6216}
6217
6218int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
6219{
6220 struct sg_table *pages;
6221 int err;
6222
6223 if (align > obj->base.size)
6224 return -EINVAL;
6225
6226 if (obj->ops == &i915_gem_phys_ops)
6227 return 0;
6228
6229 if (obj->ops != &i915_gem_object_ops)
6230 return -EINVAL;
6231
6232 err = i915_gem_object_unbind(obj);
6233 if (err)
6234 return err;
6235
6236 mutex_lock(&obj->mm.lock);
6237
6238 if (obj->mm.madv != I915_MADV_WILLNEED) {
6239 err = -EFAULT;
6240 goto err_unlock;
6241 }
6242
6243 if (obj->mm.quirked) {
6244 err = -EFAULT;
6245 goto err_unlock;
6246 }
6247
6248 if (obj->mm.mapping) {
6249 err = -EBUSY;
6250 goto err_unlock;
6251 }
6252
6253 pages = __i915_gem_object_unset_pages(obj);
6254
6255 obj->ops = &i915_gem_phys_ops;
6256
6257 err = ____i915_gem_object_get_pages(obj);
6258 if (err)
6259 goto err_xfer;
6260
6261
6262 __i915_gem_object_pin_pages(obj);
6263
6264 if (!IS_ERR_OR_NULL(pages))
6265 i915_gem_object_ops.put_pages(obj, pages);
6266 mutex_unlock(&obj->mm.lock);
6267 return 0;
6268
6269err_xfer:
6270 obj->ops = &i915_gem_object_ops;
6271 if (!IS_ERR_OR_NULL(pages)) {
6272 unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl);
6273
6274 __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
6275 }
6276err_unlock:
6277 mutex_unlock(&obj->mm.lock);
6278 return err;
6279}
6280
6281#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
6282#include "selftests/scatterlist.c"
6283#include "selftests/mock_gem_device.c"
6284#include "selftests/huge_gem_object.c"
6285#include "selftests/huge_pages.c"
6286#include "selftests/i915_gem_object.c"
6287#include "selftests/i915_gem_coherency.c"
6288#include "selftests/i915_gem.c"
6289#endif
6290