1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/dma_remapping.h>
30#include <linux/reservation.h>
31#include <linux/sync_file.h>
32#include <linux/uaccess.h>
33
34#include <drm/drmP.h>
35#include <drm/i915_drm.h>
36
37#include "i915_drv.h"
38#include "i915_gem_clflush.h"
39#include "i915_trace.h"
40#include "intel_drv.h"
41#include "intel_frontbuffer.h"
42
43enum {
44 FORCE_CPU_RELOC = 1,
45 FORCE_GTT_RELOC,
46 FORCE_GPU_RELOC,
47#define DBG_FORCE_RELOC 0
48};
49
50#define __EXEC_OBJECT_HAS_REF BIT(31)
51#define __EXEC_OBJECT_HAS_PIN BIT(30)
52#define __EXEC_OBJECT_HAS_FENCE BIT(29)
53#define __EXEC_OBJECT_NEEDS_MAP BIT(28)
54#define __EXEC_OBJECT_NEEDS_BIAS BIT(27)
55#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 27)
56#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
57
58#define __EXEC_HAS_RELOC BIT(31)
59#define __EXEC_VALIDATED BIT(30)
60#define UPDATE PIN_OFFSET_FIXED
61
62#define BATCH_OFFSET_BIAS (256*1024)
63
64#define __I915_EXEC_ILLEGAL_FLAGS \
65 (__I915_EXEC_UNKNOWN_FLAGS | I915_EXEC_CONSTANTS_MASK)
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189struct i915_execbuffer {
190 struct drm_i915_private *i915;
191 struct drm_file *file;
192 struct drm_i915_gem_execbuffer2 *args;
193 struct drm_i915_gem_exec_object2 *exec;
194
195 struct intel_engine_cs *engine;
196 struct i915_gem_context *ctx;
197 struct i915_address_space *vm;
198
199 struct drm_i915_gem_request *request;
200 struct i915_vma *batch;
201
202
203 unsigned int buffer_count;
204
205
206 struct list_head unbound;
207
208
209 struct list_head relocs;
210
211
212
213
214
215
216 struct reloc_cache {
217 struct drm_mm_node node;
218 unsigned long vaddr;
219 unsigned long page;
220 unsigned int gen;
221 bool use_64bit_reloc : 1;
222 bool has_llc : 1;
223 bool has_fence : 1;
224 bool needs_unfenced : 1;
225
226 struct drm_i915_gem_request *rq;
227 u32 *rq_cmd;
228 unsigned int rq_size;
229 } reloc_cache;
230
231 u64 invalid_flags;
232 u32 context_flags;
233
234 u32 batch_start_offset;
235 u32 batch_len;
236 u32 batch_flags;
237
238
239
240
241
242
243 int lut_size;
244 struct hlist_head *buckets;
245};
246
247
248
249
250
251
252#define __exec_to_vma(ee) (ee)->rsvd2
253#define exec_to_vma(ee) u64_to_ptr(struct i915_vma, __exec_to_vma(ee))
254
255
256
257
258
259
260
261
262
263#define GEN8_HIGH_ADDRESS_BIT 47
264static inline u64 gen8_canonical_addr(u64 address)
265{
266 return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
267}
268
269static inline u64 gen8_noncanonical_addr(u64 address)
270{
271 return address & GENMASK_ULL(GEN8_HIGH_ADDRESS_BIT, 0);
272}
273
274static int eb_create(struct i915_execbuffer *eb)
275{
276 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
277 unsigned int size = 1 + ilog2(eb->buffer_count);
278
279
280
281
282
283
284
285
286
287
288
289
290 do {
291 unsigned int flags;
292
293
294
295
296
297
298
299 flags = GFP_TEMPORARY;
300 if (size > 1)
301 flags |= __GFP_NORETRY | __GFP_NOWARN;
302
303 eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
304 flags);
305 if (eb->buckets)
306 break;
307 } while (--size);
308
309 if (unlikely(!size))
310 return -ENOMEM;
311
312 eb->lut_size = size;
313 } else {
314 eb->lut_size = -eb->buffer_count;
315 }
316
317 return 0;
318}
319
320static bool
321eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
322 const struct i915_vma *vma)
323{
324 if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
325 return true;
326
327 if (vma->node.size < entry->pad_to_size)
328 return true;
329
330 if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
331 return true;
332
333 if (entry->flags & EXEC_OBJECT_PINNED &&
334 vma->node.start != entry->offset)
335 return true;
336
337 if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
338 vma->node.start < BATCH_OFFSET_BIAS)
339 return true;
340
341 if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
342 (vma->node.start + vma->node.size - 1) >> 32)
343 return true;
344
345 return false;
346}
347
348static inline void
349eb_pin_vma(struct i915_execbuffer *eb,
350 struct drm_i915_gem_exec_object2 *entry,
351 struct i915_vma *vma)
352{
353 u64 flags;
354
355 if (vma->node.size)
356 flags = vma->node.start;
357 else
358 flags = entry->offset & PIN_OFFSET_MASK;
359
360 flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
361 if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_GTT))
362 flags |= PIN_GLOBAL;
363
364 if (unlikely(i915_vma_pin(vma, 0, 0, flags)))
365 return;
366
367 if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
368 if (unlikely(i915_vma_get_fence(vma))) {
369 i915_vma_unpin(vma);
370 return;
371 }
372
373 if (i915_vma_pin_fence(vma))
374 entry->flags |= __EXEC_OBJECT_HAS_FENCE;
375 }
376
377 entry->flags |= __EXEC_OBJECT_HAS_PIN;
378}
379
380static inline void
381__eb_unreserve_vma(struct i915_vma *vma,
382 const struct drm_i915_gem_exec_object2 *entry)
383{
384 GEM_BUG_ON(!(entry->flags & __EXEC_OBJECT_HAS_PIN));
385
386 if (unlikely(entry->flags & __EXEC_OBJECT_HAS_FENCE))
387 i915_vma_unpin_fence(vma);
388
389 __i915_vma_unpin(vma);
390}
391
392static inline void
393eb_unreserve_vma(struct i915_vma *vma,
394 struct drm_i915_gem_exec_object2 *entry)
395{
396 if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
397 return;
398
399 __eb_unreserve_vma(vma, entry);
400 entry->flags &= ~__EXEC_OBJECT_RESERVED;
401}
402
403static int
404eb_validate_vma(struct i915_execbuffer *eb,
405 struct drm_i915_gem_exec_object2 *entry,
406 struct i915_vma *vma)
407{
408 if (unlikely(entry->flags & eb->invalid_flags))
409 return -EINVAL;
410
411 if (unlikely(entry->alignment && !is_power_of_2(entry->alignment)))
412 return -EINVAL;
413
414
415
416
417
418 if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
419 entry->offset != gen8_canonical_addr(entry->offset & PAGE_MASK)))
420 return -EINVAL;
421
422
423 if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
424 if (unlikely(offset_in_page(entry->pad_to_size)))
425 return -EINVAL;
426 } else {
427 entry->pad_to_size = 0;
428 }
429
430 if (unlikely(vma->exec_entry)) {
431 DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
432 entry->handle, (int)(entry - eb->exec));
433 return -EINVAL;
434 }
435
436
437
438
439
440
441 entry->offset = gen8_noncanonical_addr(entry->offset);
442
443 return 0;
444}
445
446static int
447eb_add_vma(struct i915_execbuffer *eb,
448 struct drm_i915_gem_exec_object2 *entry,
449 struct i915_vma *vma)
450{
451 int err;
452
453 GEM_BUG_ON(i915_vma_is_closed(vma));
454
455 if (!(eb->args->flags & __EXEC_VALIDATED)) {
456 err = eb_validate_vma(eb, entry, vma);
457 if (unlikely(err))
458 return err;
459 }
460
461 if (eb->lut_size > 0) {
462 vma->exec_handle = entry->handle;
463 hlist_add_head(&vma->exec_node,
464 &eb->buckets[hash_32(entry->handle,
465 eb->lut_size)]);
466 }
467
468 if (entry->relocation_count)
469 list_add_tail(&vma->reloc_link, &eb->relocs);
470
471 if (!eb->reloc_cache.has_fence) {
472 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
473 } else {
474 if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
475 eb->reloc_cache.needs_unfenced) &&
476 i915_gem_object_is_tiled(vma->obj))
477 entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
478 }
479
480 if (!(entry->flags & EXEC_OBJECT_PINNED))
481 entry->flags |= eb->context_flags;
482
483
484
485
486
487
488
489 vma->exec_entry = entry;
490 __exec_to_vma(entry) = (uintptr_t)vma;
491
492 err = 0;
493 eb_pin_vma(eb, entry, vma);
494 if (eb_vma_misplaced(entry, vma)) {
495 eb_unreserve_vma(vma, entry);
496
497 list_add_tail(&vma->exec_link, &eb->unbound);
498 if (drm_mm_node_allocated(&vma->node))
499 err = i915_vma_unbind(vma);
500 } else {
501 if (entry->offset != vma->node.start) {
502 entry->offset = vma->node.start | UPDATE;
503 eb->args->flags |= __EXEC_HAS_RELOC;
504 }
505 }
506 return err;
507}
508
509static inline int use_cpu_reloc(const struct reloc_cache *cache,
510 const struct drm_i915_gem_object *obj)
511{
512 if (!i915_gem_object_has_struct_page(obj))
513 return false;
514
515 if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
516 return true;
517
518 if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
519 return false;
520
521 return (cache->has_llc ||
522 obj->cache_dirty ||
523 obj->cache_level != I915_CACHE_NONE);
524}
525
526static int eb_reserve_vma(const struct i915_execbuffer *eb,
527 struct i915_vma *vma)
528{
529 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
530 u64 flags;
531 int err;
532
533 flags = PIN_USER | PIN_NONBLOCK;
534 if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
535 flags |= PIN_GLOBAL;
536
537
538
539
540
541 if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
542 flags |= PIN_ZONE_4G;
543
544 if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
545 flags |= PIN_MAPPABLE;
546
547 if (entry->flags & EXEC_OBJECT_PINNED) {
548 flags |= entry->offset | PIN_OFFSET_FIXED;
549 flags &= ~PIN_NONBLOCK;
550 } else if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) {
551 flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
552 }
553
554 err = i915_vma_pin(vma, entry->pad_to_size, entry->alignment, flags);
555 if (err)
556 return err;
557
558 if (entry->offset != vma->node.start) {
559 entry->offset = vma->node.start | UPDATE;
560 eb->args->flags |= __EXEC_HAS_RELOC;
561 }
562
563 if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
564 err = i915_vma_get_fence(vma);
565 if (unlikely(err)) {
566 i915_vma_unpin(vma);
567 return err;
568 }
569
570 if (i915_vma_pin_fence(vma))
571 entry->flags |= __EXEC_OBJECT_HAS_FENCE;
572 }
573
574 entry->flags |= __EXEC_OBJECT_HAS_PIN;
575 GEM_BUG_ON(eb_vma_misplaced(entry, vma));
576
577 return 0;
578}
579
580static int eb_reserve(struct i915_execbuffer *eb)
581{
582 const unsigned int count = eb->buffer_count;
583 struct list_head last;
584 struct i915_vma *vma;
585 unsigned int i, pass;
586 int err;
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602 pass = 0;
603 err = 0;
604 do {
605 list_for_each_entry(vma, &eb->unbound, exec_link) {
606 err = eb_reserve_vma(eb, vma);
607 if (err)
608 break;
609 }
610 if (err != -ENOSPC)
611 return err;
612
613
614 INIT_LIST_HEAD(&eb->unbound);
615 INIT_LIST_HEAD(&last);
616 for (i = 0; i < count; i++) {
617 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
618
619 if (entry->flags & EXEC_OBJECT_PINNED &&
620 entry->flags & __EXEC_OBJECT_HAS_PIN)
621 continue;
622
623 vma = exec_to_vma(entry);
624 eb_unreserve_vma(vma, entry);
625
626 if (entry->flags & EXEC_OBJECT_PINNED)
627 list_add(&vma->exec_link, &eb->unbound);
628 else if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
629 list_add_tail(&vma->exec_link, &eb->unbound);
630 else
631 list_add_tail(&vma->exec_link, &last);
632 }
633 list_splice_tail(&last, &eb->unbound);
634
635 switch (pass++) {
636 case 0:
637 break;
638
639 case 1:
640
641 err = i915_gem_evict_vm(eb->vm);
642 if (err)
643 return err;
644 break;
645
646 default:
647 return -ENOSPC;
648 }
649 } while (1);
650}
651
652static inline struct hlist_head *
653ht_head(const struct i915_gem_context_vma_lut *lut, u32 handle)
654{
655 return &lut->ht[hash_32(handle, lut->ht_bits)];
656}
657
658static inline bool
659ht_needs_resize(const struct i915_gem_context_vma_lut *lut)
660{
661 return (4*lut->ht_count > 3*lut->ht_size ||
662 4*lut->ht_count + 1 < lut->ht_size);
663}
664
665static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
666{
667 if (eb->args->flags & I915_EXEC_BATCH_FIRST)
668 return 0;
669 else
670 return eb->buffer_count - 1;
671}
672
673static int eb_select_context(struct i915_execbuffer *eb)
674{
675 struct i915_gem_context *ctx;
676
677 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
678 if (unlikely(IS_ERR(ctx)))
679 return PTR_ERR(ctx);
680
681 if (unlikely(i915_gem_context_is_banned(ctx))) {
682 DRM_DEBUG("Context %u tried to submit while banned\n",
683 ctx->user_handle);
684 return -EIO;
685 }
686
687 eb->ctx = i915_gem_context_get(ctx);
688 eb->vm = ctx->ppgtt ? &ctx->ppgtt->base : &eb->i915->ggtt.base;
689
690 eb->context_flags = 0;
691 if (ctx->flags & CONTEXT_NO_ZEROMAP)
692 eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS;
693
694 return 0;
695}
696
697static int eb_lookup_vmas(struct i915_execbuffer *eb)
698{
699#define INTERMEDIATE BIT(0)
700 const unsigned int count = eb->buffer_count;
701 struct i915_gem_context_vma_lut *lut = &eb->ctx->vma_lut;
702 struct i915_vma *vma;
703 struct idr *idr;
704 unsigned int i;
705 int slow_pass = -1;
706 int err;
707
708 INIT_LIST_HEAD(&eb->relocs);
709 INIT_LIST_HEAD(&eb->unbound);
710
711 if (unlikely(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS))
712 flush_work(&lut->resize);
713 GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
714
715 for (i = 0; i < count; i++) {
716 __exec_to_vma(&eb->exec[i]) = 0;
717
718 hlist_for_each_entry(vma,
719 ht_head(lut, eb->exec[i].handle),
720 ctx_node) {
721 if (vma->ctx_handle != eb->exec[i].handle)
722 continue;
723
724 err = eb_add_vma(eb, &eb->exec[i], vma);
725 if (unlikely(err))
726 return err;
727
728 goto next_vma;
729 }
730
731 if (slow_pass < 0)
732 slow_pass = i;
733next_vma: ;
734 }
735
736 if (slow_pass < 0)
737 goto out;
738
739 spin_lock(&eb->file->table_lock);
740
741
742
743
744 idr = &eb->file->object_idr;
745 for (i = slow_pass; i < count; i++) {
746 struct drm_i915_gem_object *obj;
747
748 if (__exec_to_vma(&eb->exec[i]))
749 continue;
750
751 obj = to_intel_bo(idr_find(idr, eb->exec[i].handle));
752 if (unlikely(!obj)) {
753 spin_unlock(&eb->file->table_lock);
754 DRM_DEBUG("Invalid object handle %d at index %d\n",
755 eb->exec[i].handle, i);
756 err = -ENOENT;
757 goto err;
758 }
759
760 __exec_to_vma(&eb->exec[i]) = INTERMEDIATE | (uintptr_t)obj;
761 }
762 spin_unlock(&eb->file->table_lock);
763
764 for (i = slow_pass; i < count; i++) {
765 struct drm_i915_gem_object *obj;
766
767 if (!(__exec_to_vma(&eb->exec[i]) & INTERMEDIATE))
768 continue;
769
770
771
772
773
774
775
776
777
778 obj = u64_to_ptr(typeof(*obj),
779 __exec_to_vma(&eb->exec[i]) & ~INTERMEDIATE);
780 vma = i915_vma_instance(obj, eb->vm, NULL);
781 if (unlikely(IS_ERR(vma))) {
782 DRM_DEBUG("Failed to lookup VMA\n");
783 err = PTR_ERR(vma);
784 goto err;
785 }
786
787
788 if (!vma->ctx) {
789 vma->ctx = eb->ctx;
790 vma->ctx_handle = eb->exec[i].handle;
791 hlist_add_head(&vma->ctx_node,
792 ht_head(lut, eb->exec[i].handle));
793 lut->ht_count++;
794 lut->ht_size |= I915_CTX_RESIZE_IN_PROGRESS;
795 if (i915_vma_is_ggtt(vma)) {
796 GEM_BUG_ON(obj->vma_hashed);
797 obj->vma_hashed = vma;
798 }
799
800 i915_vma_get(vma);
801 }
802
803 err = eb_add_vma(eb, &eb->exec[i], vma);
804 if (unlikely(err))
805 goto err;
806
807
808 if (vma->ctx != eb->ctx) {
809 i915_vma_get(vma);
810 eb->exec[i].flags |= __EXEC_OBJECT_HAS_REF;
811 }
812 }
813
814 if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
815 if (ht_needs_resize(lut))
816 queue_work(system_highpri_wq, &lut->resize);
817 else
818 lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
819 }
820
821out:
822
823 i = eb_batch_index(eb);
824 eb->batch = exec_to_vma(&eb->exec[i]);
825
826
827
828
829
830
831
832
833
834
835 if (!(eb->exec[i].flags & EXEC_OBJECT_PINNED))
836 eb->exec[i].flags |= __EXEC_OBJECT_NEEDS_BIAS;
837 if (eb->reloc_cache.has_fence)
838 eb->exec[i].flags |= EXEC_OBJECT_NEEDS_FENCE;
839
840 eb->args->flags |= __EXEC_VALIDATED;
841 return eb_reserve(eb);
842
843err:
844 for (i = slow_pass; i < count; i++) {
845 if (__exec_to_vma(&eb->exec[i]) & INTERMEDIATE)
846 __exec_to_vma(&eb->exec[i]) = 0;
847 }
848 lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
849 return err;
850#undef INTERMEDIATE
851}
852
853static struct i915_vma *
854eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
855{
856 if (eb->lut_size < 0) {
857 if (handle >= -eb->lut_size)
858 return NULL;
859 return exec_to_vma(&eb->exec[handle]);
860 } else {
861 struct hlist_head *head;
862 struct i915_vma *vma;
863
864 head = &eb->buckets[hash_32(handle, eb->lut_size)];
865 hlist_for_each_entry(vma, head, exec_node) {
866 if (vma->exec_handle == handle)
867 return vma;
868 }
869 return NULL;
870 }
871}
872
873static void eb_release_vmas(const struct i915_execbuffer *eb)
874{
875 const unsigned int count = eb->buffer_count;
876 unsigned int i;
877
878 for (i = 0; i < count; i++) {
879 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
880 struct i915_vma *vma = exec_to_vma(entry);
881
882 if (!vma)
883 continue;
884
885 GEM_BUG_ON(vma->exec_entry != entry);
886 vma->exec_entry = NULL;
887 __exec_to_vma(entry) = 0;
888
889 if (entry->flags & __EXEC_OBJECT_HAS_PIN)
890 __eb_unreserve_vma(vma, entry);
891
892 if (entry->flags & __EXEC_OBJECT_HAS_REF)
893 i915_vma_put(vma);
894
895 entry->flags &=
896 ~(__EXEC_OBJECT_RESERVED | __EXEC_OBJECT_HAS_REF);
897 }
898}
899
900static void eb_reset_vmas(const struct i915_execbuffer *eb)
901{
902 eb_release_vmas(eb);
903 if (eb->lut_size > 0)
904 memset(eb->buckets, 0,
905 sizeof(struct hlist_head) << eb->lut_size);
906}
907
908static void eb_destroy(const struct i915_execbuffer *eb)
909{
910 GEM_BUG_ON(eb->reloc_cache.rq);
911
912 if (eb->lut_size > 0)
913 kfree(eb->buckets);
914}
915
916static inline u64
917relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
918 const struct i915_vma *target)
919{
920 return gen8_canonical_addr((int)reloc->delta + target->node.start);
921}
922
923static void reloc_cache_init(struct reloc_cache *cache,
924 struct drm_i915_private *i915)
925{
926 cache->page = -1;
927 cache->vaddr = 0;
928
929 cache->gen = INTEL_GEN(i915);
930 cache->has_llc = HAS_LLC(i915);
931 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
932 cache->has_fence = cache->gen < 4;
933 cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
934 cache->node.allocated = false;
935 cache->rq = NULL;
936 cache->rq_size = 0;
937}
938
939static inline void *unmask_page(unsigned long p)
940{
941 return (void *)(uintptr_t)(p & PAGE_MASK);
942}
943
944static inline unsigned int unmask_flags(unsigned long p)
945{
946 return p & ~PAGE_MASK;
947}
948
949#define KMAP 0x4
950
951static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
952{
953 struct drm_i915_private *i915 =
954 container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
955 return &i915->ggtt;
956}
957
958static void reloc_gpu_flush(struct reloc_cache *cache)
959{
960 GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32));
961 cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;
962 i915_gem_object_unpin_map(cache->rq->batch->obj);
963 i915_gem_chipset_flush(cache->rq->i915);
964
965 __i915_add_request(cache->rq, true);
966 cache->rq = NULL;
967}
968
969static void reloc_cache_reset(struct reloc_cache *cache)
970{
971 void *vaddr;
972
973 if (cache->rq)
974 reloc_gpu_flush(cache);
975
976 if (!cache->vaddr)
977 return;
978
979 vaddr = unmask_page(cache->vaddr);
980 if (cache->vaddr & KMAP) {
981 if (cache->vaddr & CLFLUSH_AFTER)
982 mb();
983
984 kunmap_atomic(vaddr);
985 i915_gem_obj_finish_shmem_access((struct drm_i915_gem_object *)cache->node.mm);
986 } else {
987 wmb();
988 io_mapping_unmap_atomic((void __iomem *)vaddr);
989 if (cache->node.allocated) {
990 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
991
992 ggtt->base.clear_range(&ggtt->base,
993 cache->node.start,
994 cache->node.size);
995 drm_mm_remove_node(&cache->node);
996 } else {
997 i915_vma_unpin((struct i915_vma *)cache->node.mm);
998 }
999 }
1000
1001 cache->vaddr = 0;
1002 cache->page = -1;
1003}
1004
1005static void *reloc_kmap(struct drm_i915_gem_object *obj,
1006 struct reloc_cache *cache,
1007 unsigned long page)
1008{
1009 void *vaddr;
1010
1011 if (cache->vaddr) {
1012 kunmap_atomic(unmask_page(cache->vaddr));
1013 } else {
1014 unsigned int flushes;
1015 int err;
1016
1017 err = i915_gem_obj_prepare_shmem_write(obj, &flushes);
1018 if (err)
1019 return ERR_PTR(err);
1020
1021 BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
1022 BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
1023
1024 cache->vaddr = flushes | KMAP;
1025 cache->node.mm = (void *)obj;
1026 if (flushes)
1027 mb();
1028 }
1029
1030 vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page));
1031 cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
1032 cache->page = page;
1033
1034 return vaddr;
1035}
1036
1037static void *reloc_iomap(struct drm_i915_gem_object *obj,
1038 struct reloc_cache *cache,
1039 unsigned long page)
1040{
1041 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1042 unsigned long offset;
1043 void *vaddr;
1044
1045 if (cache->vaddr) {
1046 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
1047 } else {
1048 struct i915_vma *vma;
1049 int err;
1050
1051 if (use_cpu_reloc(cache, obj))
1052 return NULL;
1053
1054 err = i915_gem_object_set_to_gtt_domain(obj, true);
1055 if (err)
1056 return ERR_PTR(err);
1057
1058 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1059 PIN_MAPPABLE | PIN_NONBLOCK);
1060 if (IS_ERR(vma)) {
1061 memset(&cache->node, 0, sizeof(cache->node));
1062 err = drm_mm_insert_node_in_range
1063 (&ggtt->base.mm, &cache->node,
1064 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
1065 0, ggtt->mappable_end,
1066 DRM_MM_INSERT_LOW);
1067 if (err)
1068 return NULL;
1069 } else {
1070 err = i915_vma_put_fence(vma);
1071 if (err) {
1072 i915_vma_unpin(vma);
1073 return ERR_PTR(err);
1074 }
1075
1076 cache->node.start = vma->node.start;
1077 cache->node.mm = (void *)vma;
1078 }
1079 }
1080
1081 offset = cache->node.start;
1082 if (cache->node.allocated) {
1083 wmb();
1084 ggtt->base.insert_page(&ggtt->base,
1085 i915_gem_object_get_dma_address(obj, page),
1086 offset, I915_CACHE_NONE, 0);
1087 } else {
1088 offset += page << PAGE_SHIFT;
1089 }
1090
1091 vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->mappable,
1092 offset);
1093 cache->page = page;
1094 cache->vaddr = (unsigned long)vaddr;
1095
1096 return vaddr;
1097}
1098
1099static void *reloc_vaddr(struct drm_i915_gem_object *obj,
1100 struct reloc_cache *cache,
1101 unsigned long page)
1102{
1103 void *vaddr;
1104
1105 if (cache->page == page) {
1106 vaddr = unmask_page(cache->vaddr);
1107 } else {
1108 vaddr = NULL;
1109 if ((cache->vaddr & KMAP) == 0)
1110 vaddr = reloc_iomap(obj, cache, page);
1111 if (!vaddr)
1112 vaddr = reloc_kmap(obj, cache, page);
1113 }
1114
1115 return vaddr;
1116}
1117
1118static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
1119{
1120 if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
1121 if (flushes & CLFLUSH_BEFORE) {
1122 clflushopt(addr);
1123 mb();
1124 }
1125
1126 *addr = value;
1127
1128
1129
1130
1131
1132
1133
1134
1135 if (flushes & CLFLUSH_AFTER)
1136 clflushopt(addr);
1137 } else
1138 *addr = value;
1139}
1140
1141static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
1142 struct i915_vma *vma,
1143 unsigned int len)
1144{
1145 struct reloc_cache *cache = &eb->reloc_cache;
1146 struct drm_i915_gem_object *obj;
1147 struct drm_i915_gem_request *rq;
1148 struct i915_vma *batch;
1149 u32 *cmd;
1150 int err;
1151
1152 GEM_BUG_ON(vma->obj->base.write_domain & I915_GEM_DOMAIN_CPU);
1153
1154 obj = i915_gem_batch_pool_get(&eb->engine->batch_pool, PAGE_SIZE);
1155 if (IS_ERR(obj))
1156 return PTR_ERR(obj);
1157
1158 cmd = i915_gem_object_pin_map(obj,
1159 cache->has_llc ? I915_MAP_WB : I915_MAP_WC);
1160 i915_gem_object_unpin_pages(obj);
1161 if (IS_ERR(cmd))
1162 return PTR_ERR(cmd);
1163
1164 err = i915_gem_object_set_to_wc_domain(obj, false);
1165 if (err)
1166 goto err_unmap;
1167
1168 batch = i915_vma_instance(obj, vma->vm, NULL);
1169 if (IS_ERR(batch)) {
1170 err = PTR_ERR(batch);
1171 goto err_unmap;
1172 }
1173
1174 err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK);
1175 if (err)
1176 goto err_unmap;
1177
1178 rq = i915_gem_request_alloc(eb->engine, eb->ctx);
1179 if (IS_ERR(rq)) {
1180 err = PTR_ERR(rq);
1181 goto err_unpin;
1182 }
1183
1184 err = i915_gem_request_await_object(rq, vma->obj, true);
1185 if (err)
1186 goto err_request;
1187
1188 err = eb->engine->emit_flush(rq, EMIT_INVALIDATE);
1189 if (err)
1190 goto err_request;
1191
1192 err = i915_switch_context(rq);
1193 if (err)
1194 goto err_request;
1195
1196 err = eb->engine->emit_bb_start(rq,
1197 batch->node.start, PAGE_SIZE,
1198 cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
1199 if (err)
1200 goto err_request;
1201
1202 GEM_BUG_ON(!reservation_object_test_signaled_rcu(batch->resv, true));
1203 i915_vma_move_to_active(batch, rq, 0);
1204 reservation_object_lock(batch->resv, NULL);
1205 reservation_object_add_excl_fence(batch->resv, &rq->fence);
1206 reservation_object_unlock(batch->resv);
1207 i915_vma_unpin(batch);
1208
1209 i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1210 reservation_object_lock(vma->resv, NULL);
1211 reservation_object_add_excl_fence(vma->resv, &rq->fence);
1212 reservation_object_unlock(vma->resv);
1213
1214 rq->batch = batch;
1215
1216 cache->rq = rq;
1217 cache->rq_cmd = cmd;
1218 cache->rq_size = 0;
1219
1220
1221 return 0;
1222
1223err_request:
1224 i915_add_request(rq);
1225err_unpin:
1226 i915_vma_unpin(batch);
1227err_unmap:
1228 i915_gem_object_unpin_map(obj);
1229 return err;
1230}
1231
1232static u32 *reloc_gpu(struct i915_execbuffer *eb,
1233 struct i915_vma *vma,
1234 unsigned int len)
1235{
1236 struct reloc_cache *cache = &eb->reloc_cache;
1237 u32 *cmd;
1238
1239 if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1))
1240 reloc_gpu_flush(cache);
1241
1242 if (unlikely(!cache->rq)) {
1243 int err;
1244
1245 err = __reloc_gpu_alloc(eb, vma, len);
1246 if (unlikely(err))
1247 return ERR_PTR(err);
1248 }
1249
1250 cmd = cache->rq_cmd + cache->rq_size;
1251 cache->rq_size += len;
1252
1253 return cmd;
1254}
1255
1256static u64
1257relocate_entry(struct i915_vma *vma,
1258 const struct drm_i915_gem_relocation_entry *reloc,
1259 struct i915_execbuffer *eb,
1260 const struct i915_vma *target)
1261{
1262 u64 offset = reloc->offset;
1263 u64 target_offset = relocation_target(reloc, target);
1264 bool wide = eb->reloc_cache.use_64bit_reloc;
1265 void *vaddr;
1266
1267 if (!eb->reloc_cache.vaddr &&
1268 (DBG_FORCE_RELOC == FORCE_GPU_RELOC ||
1269 !reservation_object_test_signaled_rcu(vma->resv, true))) {
1270 const unsigned int gen = eb->reloc_cache.gen;
1271 unsigned int len;
1272 u32 *batch;
1273 u64 addr;
1274
1275 if (wide)
1276 len = offset & 7 ? 8 : 5;
1277 else if (gen >= 4)
1278 len = 4;
1279 else if (gen >= 3)
1280 len = 3;
1281 else
1282 goto repeat;
1283
1284 batch = reloc_gpu(eb, vma, len);
1285 if (IS_ERR(batch))
1286 goto repeat;
1287
1288 addr = gen8_canonical_addr(vma->node.start + offset);
1289 if (wide) {
1290 if (offset & 7) {
1291 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1292 *batch++ = lower_32_bits(addr);
1293 *batch++ = upper_32_bits(addr);
1294 *batch++ = lower_32_bits(target_offset);
1295
1296 addr = gen8_canonical_addr(addr + 4);
1297
1298 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1299 *batch++ = lower_32_bits(addr);
1300 *batch++ = upper_32_bits(addr);
1301 *batch++ = upper_32_bits(target_offset);
1302 } else {
1303 *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1;
1304 *batch++ = lower_32_bits(addr);
1305 *batch++ = upper_32_bits(addr);
1306 *batch++ = lower_32_bits(target_offset);
1307 *batch++ = upper_32_bits(target_offset);
1308 }
1309 } else if (gen >= 6) {
1310 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1311 *batch++ = 0;
1312 *batch++ = addr;
1313 *batch++ = target_offset;
1314 } else if (gen >= 4) {
1315 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1316 *batch++ = 0;
1317 *batch++ = addr;
1318 *batch++ = target_offset;
1319 } else {
1320 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
1321 *batch++ = addr;
1322 *batch++ = target_offset;
1323 }
1324
1325 goto out;
1326 }
1327
1328repeat:
1329 vaddr = reloc_vaddr(vma->obj, &eb->reloc_cache, offset >> PAGE_SHIFT);
1330 if (IS_ERR(vaddr))
1331 return PTR_ERR(vaddr);
1332
1333 clflush_write32(vaddr + offset_in_page(offset),
1334 lower_32_bits(target_offset),
1335 eb->reloc_cache.vaddr);
1336
1337 if (wide) {
1338 offset += sizeof(u32);
1339 target_offset >>= 32;
1340 wide = false;
1341 goto repeat;
1342 }
1343
1344out:
1345 return target->node.start | UPDATE;
1346}
1347
1348static u64
1349eb_relocate_entry(struct i915_execbuffer *eb,
1350 struct i915_vma *vma,
1351 const struct drm_i915_gem_relocation_entry *reloc)
1352{
1353 struct i915_vma *target;
1354 int err;
1355
1356
1357 target = eb_get_vma(eb, reloc->target_handle);
1358 if (unlikely(!target))
1359 return -ENOENT;
1360
1361
1362 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
1363 DRM_DEBUG("reloc with multiple write domains: "
1364 "target %d offset %d "
1365 "read %08x write %08x",
1366 reloc->target_handle,
1367 (int) reloc->offset,
1368 reloc->read_domains,
1369 reloc->write_domain);
1370 return -EINVAL;
1371 }
1372 if (unlikely((reloc->write_domain | reloc->read_domains)
1373 & ~I915_GEM_GPU_DOMAINS)) {
1374 DRM_DEBUG("reloc with read/write non-GPU domains: "
1375 "target %d offset %d "
1376 "read %08x write %08x",
1377 reloc->target_handle,
1378 (int) reloc->offset,
1379 reloc->read_domains,
1380 reloc->write_domain);
1381 return -EINVAL;
1382 }
1383
1384 if (reloc->write_domain) {
1385 target->exec_entry->flags |= EXEC_OBJECT_WRITE;
1386
1387
1388
1389
1390
1391
1392
1393 if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
1394 IS_GEN6(eb->i915)) {
1395 err = i915_vma_bind(target, target->obj->cache_level,
1396 PIN_GLOBAL);
1397 if (WARN_ONCE(err,
1398 "Unexpected failure to bind target VMA!"))
1399 return err;
1400 }
1401 }
1402
1403
1404
1405
1406
1407 if (!DBG_FORCE_RELOC &&
1408 gen8_canonical_addr(target->node.start) == reloc->presumed_offset)
1409 return 0;
1410
1411
1412 if (unlikely(reloc->offset >
1413 vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
1414 DRM_DEBUG("Relocation beyond object bounds: "
1415 "target %d offset %d size %d.\n",
1416 reloc->target_handle,
1417 (int)reloc->offset,
1418 (int)vma->size);
1419 return -EINVAL;
1420 }
1421 if (unlikely(reloc->offset & 3)) {
1422 DRM_DEBUG("Relocation not 4-byte aligned: "
1423 "target %d offset %d.\n",
1424 reloc->target_handle,
1425 (int)reloc->offset);
1426 return -EINVAL;
1427 }
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437 vma->exec_entry->flags &= ~EXEC_OBJECT_ASYNC;
1438
1439
1440 return relocate_entry(vma, reloc, eb, target);
1441}
1442
1443static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma)
1444{
1445#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
1446 struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
1447 struct drm_i915_gem_relocation_entry __user *urelocs;
1448 const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
1449 unsigned int remain;
1450
1451 urelocs = u64_to_user_ptr(entry->relocs_ptr);
1452 remain = entry->relocation_count;
1453 if (unlikely(remain > N_RELOC(ULONG_MAX)))
1454 return -EINVAL;
1455
1456
1457
1458
1459
1460
1461 if (unlikely(!access_ok(VERIFY_READ, urelocs, remain*sizeof(*urelocs))))
1462 return -EFAULT;
1463
1464 do {
1465 struct drm_i915_gem_relocation_entry *r = stack;
1466 unsigned int count =
1467 min_t(unsigned int, remain, ARRAY_SIZE(stack));
1468 unsigned int copied;
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478 pagefault_disable();
1479 copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
1480 pagefault_enable();
1481 if (unlikely(copied)) {
1482 remain = -EFAULT;
1483 goto out;
1484 }
1485
1486 remain -= count;
1487 do {
1488 u64 offset = eb_relocate_entry(eb, vma, r);
1489
1490 if (likely(offset == 0)) {
1491 } else if ((s64)offset < 0) {
1492 remain = (int)offset;
1493 goto out;
1494 } else {
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516 offset = gen8_canonical_addr(offset & ~UPDATE);
1517 __put_user(offset,
1518 &urelocs[r-stack].presumed_offset);
1519 }
1520 } while (r++, --count);
1521 urelocs += ARRAY_SIZE(stack);
1522 } while (remain);
1523out:
1524 reloc_cache_reset(&eb->reloc_cache);
1525 return remain;
1526}
1527
1528static int
1529eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma)
1530{
1531 const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
1532 struct drm_i915_gem_relocation_entry *relocs =
1533 u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1534 unsigned int i;
1535 int err;
1536
1537 for (i = 0; i < entry->relocation_count; i++) {
1538 u64 offset = eb_relocate_entry(eb, vma, &relocs[i]);
1539
1540 if ((s64)offset < 0) {
1541 err = (int)offset;
1542 goto err;
1543 }
1544 }
1545 err = 0;
1546err:
1547 reloc_cache_reset(&eb->reloc_cache);
1548 return err;
1549}
1550
1551static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
1552{
1553 const char __user *addr, *end;
1554 unsigned long size;
1555 char __maybe_unused c;
1556
1557 size = entry->relocation_count;
1558 if (size == 0)
1559 return 0;
1560
1561 if (size > N_RELOC(ULONG_MAX))
1562 return -EINVAL;
1563
1564 addr = u64_to_user_ptr(entry->relocs_ptr);
1565 size *= sizeof(struct drm_i915_gem_relocation_entry);
1566 if (!access_ok(VERIFY_READ, addr, size))
1567 return -EFAULT;
1568
1569 end = addr + size;
1570 for (; addr < end; addr += PAGE_SIZE) {
1571 int err = __get_user(c, addr);
1572 if (err)
1573 return err;
1574 }
1575 return __get_user(c, end - 1);
1576}
1577
1578static int eb_copy_relocations(const struct i915_execbuffer *eb)
1579{
1580 const unsigned int count = eb->buffer_count;
1581 unsigned int i;
1582 int err;
1583
1584 for (i = 0; i < count; i++) {
1585 const unsigned int nreloc = eb->exec[i].relocation_count;
1586 struct drm_i915_gem_relocation_entry __user *urelocs;
1587 struct drm_i915_gem_relocation_entry *relocs;
1588 unsigned long size;
1589 unsigned long copied;
1590
1591 if (nreloc == 0)
1592 continue;
1593
1594 err = check_relocations(&eb->exec[i]);
1595 if (err)
1596 goto err;
1597
1598 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
1599 size = nreloc * sizeof(*relocs);
1600
1601 relocs = kvmalloc_array(size, 1, GFP_TEMPORARY);
1602 if (!relocs) {
1603 kvfree(relocs);
1604 err = -ENOMEM;
1605 goto err;
1606 }
1607
1608
1609 copied = 0;
1610 do {
1611 unsigned int len =
1612 min_t(u64, BIT_ULL(31), size - copied);
1613
1614 if (__copy_from_user((char *)relocs + copied,
1615 (char *)urelocs + copied,
1616 len)) {
1617 kvfree(relocs);
1618 err = -EFAULT;
1619 goto err;
1620 }
1621
1622 copied += len;
1623 } while (copied < size);
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635 user_access_begin();
1636 for (copied = 0; copied < nreloc; copied++)
1637 unsafe_put_user(-1,
1638 &urelocs[copied].presumed_offset,
1639 end_user);
1640end_user:
1641 user_access_end();
1642
1643 eb->exec[i].relocs_ptr = (uintptr_t)relocs;
1644 }
1645
1646 return 0;
1647
1648err:
1649 while (i--) {
1650 struct drm_i915_gem_relocation_entry *relocs =
1651 u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
1652 if (eb->exec[i].relocation_count)
1653 kvfree(relocs);
1654 }
1655 return err;
1656}
1657
1658static int eb_prefault_relocations(const struct i915_execbuffer *eb)
1659{
1660 const unsigned int count = eb->buffer_count;
1661 unsigned int i;
1662
1663 if (unlikely(i915.prefault_disable))
1664 return 0;
1665
1666 for (i = 0; i < count; i++) {
1667 int err;
1668
1669 err = check_relocations(&eb->exec[i]);
1670 if (err)
1671 return err;
1672 }
1673
1674 return 0;
1675}
1676
1677static noinline int eb_relocate_slow(struct i915_execbuffer *eb)
1678{
1679 struct drm_device *dev = &eb->i915->drm;
1680 bool have_copy = false;
1681 struct i915_vma *vma;
1682 int err = 0;
1683
1684repeat:
1685 if (signal_pending(current)) {
1686 err = -ERESTARTSYS;
1687 goto out;
1688 }
1689
1690
1691 eb_reset_vmas(eb);
1692 mutex_unlock(&dev->struct_mutex);
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707 if (!err) {
1708 err = eb_prefault_relocations(eb);
1709 } else if (!have_copy) {
1710 err = eb_copy_relocations(eb);
1711 have_copy = err == 0;
1712 } else {
1713 cond_resched();
1714 err = 0;
1715 }
1716 if (err) {
1717 mutex_lock(&dev->struct_mutex);
1718 goto out;
1719 }
1720
1721
1722 flush_workqueue(eb->i915->mm.userptr_wq);
1723
1724 err = i915_mutex_lock_interruptible(dev);
1725 if (err) {
1726 mutex_lock(&dev->struct_mutex);
1727 goto out;
1728 }
1729
1730
1731 err = eb_lookup_vmas(eb);
1732 if (err)
1733 goto err;
1734
1735 list_for_each_entry(vma, &eb->relocs, reloc_link) {
1736 if (!have_copy) {
1737 pagefault_disable();
1738 err = eb_relocate_vma(eb, vma);
1739 pagefault_enable();
1740 if (err)
1741 goto repeat;
1742 } else {
1743 err = eb_relocate_vma_slow(eb, vma);
1744 if (err)
1745 goto err;
1746 }
1747 }
1748
1749
1750
1751
1752
1753
1754
1755
1756err:
1757 if (err == -EAGAIN)
1758 goto repeat;
1759
1760out:
1761 if (have_copy) {
1762 const unsigned int count = eb->buffer_count;
1763 unsigned int i;
1764
1765 for (i = 0; i < count; i++) {
1766 const struct drm_i915_gem_exec_object2 *entry =
1767 &eb->exec[i];
1768 struct drm_i915_gem_relocation_entry *relocs;
1769
1770 if (!entry->relocation_count)
1771 continue;
1772
1773 relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1774 kvfree(relocs);
1775 }
1776 }
1777
1778 return err;
1779}
1780
1781static int eb_relocate(struct i915_execbuffer *eb)
1782{
1783 if (eb_lookup_vmas(eb))
1784 goto slow;
1785
1786
1787 if (eb->args->flags & __EXEC_HAS_RELOC) {
1788 struct i915_vma *vma;
1789
1790 list_for_each_entry(vma, &eb->relocs, reloc_link) {
1791 if (eb_relocate_vma(eb, vma))
1792 goto slow;
1793 }
1794 }
1795
1796 return 0;
1797
1798slow:
1799 return eb_relocate_slow(eb);
1800}
1801
1802static void eb_export_fence(struct i915_vma *vma,
1803 struct drm_i915_gem_request *req,
1804 unsigned int flags)
1805{
1806 struct reservation_object *resv = vma->resv;
1807
1808
1809
1810
1811
1812
1813 reservation_object_lock(resv, NULL);
1814 if (flags & EXEC_OBJECT_WRITE)
1815 reservation_object_add_excl_fence(resv, &req->fence);
1816 else if (reservation_object_reserve_shared(resv) == 0)
1817 reservation_object_add_shared_fence(resv, &req->fence);
1818 reservation_object_unlock(resv);
1819}
1820
1821static int eb_move_to_gpu(struct i915_execbuffer *eb)
1822{
1823 const unsigned int count = eb->buffer_count;
1824 unsigned int i;
1825 int err;
1826
1827 for (i = 0; i < count; i++) {
1828 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
1829 struct i915_vma *vma = exec_to_vma(entry);
1830 struct drm_i915_gem_object *obj = vma->obj;
1831
1832 if (entry->flags & EXEC_OBJECT_CAPTURE) {
1833 struct i915_gem_capture_list *capture;
1834
1835 capture = kmalloc(sizeof(*capture), GFP_KERNEL);
1836 if (unlikely(!capture))
1837 return -ENOMEM;
1838
1839 capture->next = eb->request->capture_list;
1840 capture->vma = vma;
1841 eb->request->capture_list = capture;
1842 }
1843
1844 if (unlikely(obj->cache_dirty && !obj->cache_coherent)) {
1845 if (i915_gem_clflush_object(obj, 0))
1846 entry->flags &= ~EXEC_OBJECT_ASYNC;
1847 }
1848
1849 if (entry->flags & EXEC_OBJECT_ASYNC)
1850 goto skip_flushes;
1851
1852 err = i915_gem_request_await_object
1853 (eb->request, obj, entry->flags & EXEC_OBJECT_WRITE);
1854 if (err)
1855 return err;
1856
1857skip_flushes:
1858 i915_vma_move_to_active(vma, eb->request, entry->flags);
1859 __eb_unreserve_vma(vma, entry);
1860 vma->exec_entry = NULL;
1861 }
1862
1863 for (i = 0; i < count; i++) {
1864 const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
1865 struct i915_vma *vma = exec_to_vma(entry);
1866
1867 eb_export_fence(vma, eb->request, entry->flags);
1868 if (unlikely(entry->flags & __EXEC_OBJECT_HAS_REF))
1869 i915_vma_put(vma);
1870 }
1871 eb->exec = NULL;
1872
1873
1874 i915_gem_chipset_flush(eb->i915);
1875
1876
1877 return eb->engine->emit_flush(eb->request, EMIT_INVALIDATE);
1878}
1879
1880static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1881{
1882 if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
1883 return false;
1884
1885
1886 if (exec->num_cliprects || exec->cliprects_ptr)
1887 return false;
1888
1889 if (exec->DR4 == 0xffffffff) {
1890 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
1891 exec->DR4 = 0;
1892 }
1893 if (exec->DR1 || exec->DR4)
1894 return false;
1895
1896 if ((exec->batch_start_offset | exec->batch_len) & 0x7)
1897 return false;
1898
1899 return true;
1900}
1901
1902void i915_vma_move_to_active(struct i915_vma *vma,
1903 struct drm_i915_gem_request *req,
1904 unsigned int flags)
1905{
1906 struct drm_i915_gem_object *obj = vma->obj;
1907 const unsigned int idx = req->engine->id;
1908
1909 lockdep_assert_held(&req->i915->drm.struct_mutex);
1910 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920 if (!i915_vma_is_active(vma))
1921 obj->active_count++;
1922 i915_vma_set_active(vma, idx);
1923 i915_gem_active_set(&vma->last_read[idx], req);
1924 list_move_tail(&vma->vm_link, &vma->vm->active_list);
1925
1926 obj->base.write_domain = 0;
1927 if (flags & EXEC_OBJECT_WRITE) {
1928 obj->base.write_domain = I915_GEM_DOMAIN_RENDER;
1929
1930 if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
1931 i915_gem_active_set(&obj->frontbuffer_write, req);
1932
1933 obj->base.read_domains = 0;
1934 }
1935 obj->base.read_domains |= I915_GEM_GPU_DOMAINS;
1936
1937 if (flags & EXEC_OBJECT_NEEDS_FENCE)
1938 i915_gem_active_set(&vma->last_fence, req);
1939}
1940
1941static int i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
1942{
1943 u32 *cs;
1944 int i;
1945
1946 if (!IS_GEN7(req->i915) || req->engine->id != RCS) {
1947 DRM_DEBUG("sol reset is gen7/rcs only\n");
1948 return -EINVAL;
1949 }
1950
1951 cs = intel_ring_begin(req, 4 * 2 + 2);
1952 if (IS_ERR(cs))
1953 return PTR_ERR(cs);
1954
1955 *cs++ = MI_LOAD_REGISTER_IMM(4);
1956 for (i = 0; i < 4; i++) {
1957 *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
1958 *cs++ = 0;
1959 }
1960 *cs++ = MI_NOOP;
1961 intel_ring_advance(req, cs);
1962
1963 return 0;
1964}
1965
1966static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master)
1967{
1968 struct drm_i915_gem_object *shadow_batch_obj;
1969 struct i915_vma *vma;
1970 int err;
1971
1972 shadow_batch_obj = i915_gem_batch_pool_get(&eb->engine->batch_pool,
1973 PAGE_ALIGN(eb->batch_len));
1974 if (IS_ERR(shadow_batch_obj))
1975 return ERR_CAST(shadow_batch_obj);
1976
1977 err = intel_engine_cmd_parser(eb->engine,
1978 eb->batch->obj,
1979 shadow_batch_obj,
1980 eb->batch_start_offset,
1981 eb->batch_len,
1982 is_master);
1983 if (err) {
1984 if (err == -EACCES)
1985 vma = NULL;
1986 else
1987 vma = ERR_PTR(err);
1988 goto out;
1989 }
1990
1991 vma = i915_gem_object_ggtt_pin(shadow_batch_obj, NULL, 0, 0, 0);
1992 if (IS_ERR(vma))
1993 goto out;
1994
1995 vma->exec_entry =
1996 memset(&eb->exec[eb->buffer_count++],
1997 0, sizeof(*vma->exec_entry));
1998 vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
1999 __exec_to_vma(vma->exec_entry) = (uintptr_t)i915_vma_get(vma);
2000
2001out:
2002 i915_gem_object_unpin_pages(shadow_batch_obj);
2003 return vma;
2004}
2005
2006static void
2007add_to_client(struct drm_i915_gem_request *req, struct drm_file *file)
2008{
2009 req->file_priv = file->driver_priv;
2010 list_add_tail(&req->client_link, &req->file_priv->mm.request_list);
2011}
2012
2013static int eb_submit(struct i915_execbuffer *eb)
2014{
2015 int err;
2016
2017 err = eb_move_to_gpu(eb);
2018 if (err)
2019 return err;
2020
2021 err = i915_switch_context(eb->request);
2022 if (err)
2023 return err;
2024
2025 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
2026 err = i915_reset_gen7_sol_offsets(eb->request);
2027 if (err)
2028 return err;
2029 }
2030
2031 err = eb->engine->emit_bb_start(eb->request,
2032 eb->batch->node.start +
2033 eb->batch_start_offset,
2034 eb->batch_len,
2035 eb->batch_flags);
2036 if (err)
2037 return err;
2038
2039 return 0;
2040}
2041
2042
2043
2044
2045
2046static unsigned int
2047gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
2048 struct drm_file *file)
2049{
2050 struct drm_i915_file_private *file_priv = file->driver_priv;
2051
2052
2053 if ((int)file_priv->bsd_engine < 0)
2054 file_priv->bsd_engine = atomic_fetch_xor(1,
2055 &dev_priv->mm.bsd_engine_dispatch_index);
2056
2057 return file_priv->bsd_engine;
2058}
2059
2060#define I915_USER_RINGS (4)
2061
2062static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
2063 [I915_EXEC_DEFAULT] = RCS,
2064 [I915_EXEC_RENDER] = RCS,
2065 [I915_EXEC_BLT] = BCS,
2066 [I915_EXEC_BSD] = VCS,
2067 [I915_EXEC_VEBOX] = VECS
2068};
2069
2070static struct intel_engine_cs *
2071eb_select_engine(struct drm_i915_private *dev_priv,
2072 struct drm_file *file,
2073 struct drm_i915_gem_execbuffer2 *args)
2074{
2075 unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
2076 struct intel_engine_cs *engine;
2077
2078 if (user_ring_id > I915_USER_RINGS) {
2079 DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
2080 return NULL;
2081 }
2082
2083 if ((user_ring_id != I915_EXEC_BSD) &&
2084 ((args->flags & I915_EXEC_BSD_MASK) != 0)) {
2085 DRM_DEBUG("execbuf with non bsd ring but with invalid "
2086 "bsd dispatch flags: %d\n", (int)(args->flags));
2087 return NULL;
2088 }
2089
2090 if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
2091 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
2092
2093 if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
2094 bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file);
2095 } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
2096 bsd_idx <= I915_EXEC_BSD_RING2) {
2097 bsd_idx >>= I915_EXEC_BSD_SHIFT;
2098 bsd_idx--;
2099 } else {
2100 DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
2101 bsd_idx);
2102 return NULL;
2103 }
2104
2105 engine = dev_priv->engine[_VCS(bsd_idx)];
2106 } else {
2107 engine = dev_priv->engine[user_ring_map[user_ring_id]];
2108 }
2109
2110 if (!engine) {
2111 DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
2112 return NULL;
2113 }
2114
2115 return engine;
2116}
2117
2118static int
2119i915_gem_do_execbuffer(struct drm_device *dev,
2120 struct drm_file *file,
2121 struct drm_i915_gem_execbuffer2 *args,
2122 struct drm_i915_gem_exec_object2 *exec)
2123{
2124 struct i915_execbuffer eb;
2125 struct dma_fence *in_fence = NULL;
2126 struct sync_file *out_fence = NULL;
2127 int out_fence_fd = -1;
2128 int err;
2129
2130 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
2131 ~__EXEC_OBJECT_UNKNOWN_FLAGS);
2132
2133 eb.i915 = to_i915(dev);
2134 eb.file = file;
2135 eb.args = args;
2136 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
2137 args->flags |= __EXEC_HAS_RELOC;
2138 eb.exec = exec;
2139 eb.ctx = NULL;
2140 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
2141 if (USES_FULL_PPGTT(eb.i915))
2142 eb.invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
2143 reloc_cache_init(&eb.reloc_cache, eb.i915);
2144
2145 eb.buffer_count = args->buffer_count;
2146 eb.batch_start_offset = args->batch_start_offset;
2147 eb.batch_len = args->batch_len;
2148
2149 eb.batch_flags = 0;
2150 if (args->flags & I915_EXEC_SECURE) {
2151 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
2152 return -EPERM;
2153
2154 eb.batch_flags |= I915_DISPATCH_SECURE;
2155 }
2156 if (args->flags & I915_EXEC_IS_PINNED)
2157 eb.batch_flags |= I915_DISPATCH_PINNED;
2158
2159 eb.engine = eb_select_engine(eb.i915, file, args);
2160 if (!eb.engine)
2161 return -EINVAL;
2162
2163 if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
2164 if (!HAS_RESOURCE_STREAMER(eb.i915)) {
2165 DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
2166 return -EINVAL;
2167 }
2168 if (eb.engine->id != RCS) {
2169 DRM_DEBUG("RS is not available on %s\n",
2170 eb.engine->name);
2171 return -EINVAL;
2172 }
2173
2174 eb.batch_flags |= I915_DISPATCH_RS;
2175 }
2176
2177 if (args->flags & I915_EXEC_FENCE_IN) {
2178 in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
2179 if (!in_fence)
2180 return -EINVAL;
2181 }
2182
2183 if (args->flags & I915_EXEC_FENCE_OUT) {
2184 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
2185 if (out_fence_fd < 0) {
2186 err = out_fence_fd;
2187 goto err_in_fence;
2188 }
2189 }
2190
2191 err = eb_create(&eb);
2192 if (err)
2193 goto err_out_fence;
2194
2195 GEM_BUG_ON(!eb.lut_size);
2196
2197
2198
2199
2200
2201
2202
2203
2204 intel_runtime_pm_get(eb.i915);
2205 err = i915_mutex_lock_interruptible(dev);
2206 if (err)
2207 goto err_rpm;
2208
2209 err = eb_select_context(&eb);
2210 if (unlikely(err))
2211 goto err_unlock;
2212
2213 err = eb_relocate(&eb);
2214 if (err) {
2215
2216
2217
2218
2219
2220
2221
2222 args->flags &= ~__EXEC_HAS_RELOC;
2223 goto err_vma;
2224 }
2225
2226 if (unlikely(eb.batch->exec_entry->flags & EXEC_OBJECT_WRITE)) {
2227 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
2228 err = -EINVAL;
2229 goto err_vma;
2230 }
2231 if (eb.batch_start_offset > eb.batch->size ||
2232 eb.batch_len > eb.batch->size - eb.batch_start_offset) {
2233 DRM_DEBUG("Attempting to use out-of-bounds batch\n");
2234 err = -EINVAL;
2235 goto err_vma;
2236 }
2237
2238 if (eb.engine->needs_cmd_parser && eb.batch_len) {
2239 struct i915_vma *vma;
2240
2241 vma = eb_parse(&eb, drm_is_current_master(file));
2242 if (IS_ERR(vma)) {
2243 err = PTR_ERR(vma);
2244 goto err_vma;
2245 }
2246
2247 if (vma) {
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257 eb.batch_flags |= I915_DISPATCH_SECURE;
2258 eb.batch_start_offset = 0;
2259 eb.batch = vma;
2260 }
2261 }
2262
2263 if (eb.batch_len == 0)
2264 eb.batch_len = eb.batch->size - eb.batch_start_offset;
2265
2266
2267
2268
2269
2270 if (eb.batch_flags & I915_DISPATCH_SECURE) {
2271 struct i915_vma *vma;
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283 vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0);
2284 if (IS_ERR(vma)) {
2285 err = PTR_ERR(vma);
2286 goto err_vma;
2287 }
2288
2289 eb.batch = vma;
2290 }
2291
2292
2293 GEM_BUG_ON(eb.reloc_cache.rq);
2294
2295
2296 eb.request = i915_gem_request_alloc(eb.engine, eb.ctx);
2297 if (IS_ERR(eb.request)) {
2298 err = PTR_ERR(eb.request);
2299 goto err_batch_unpin;
2300 }
2301
2302 if (in_fence) {
2303 err = i915_gem_request_await_dma_fence(eb.request, in_fence);
2304 if (err < 0)
2305 goto err_request;
2306 }
2307
2308 if (out_fence_fd != -1) {
2309 out_fence = sync_file_create(&eb.request->fence);
2310 if (!out_fence) {
2311 err = -ENOMEM;
2312 goto err_request;
2313 }
2314 }
2315
2316
2317
2318
2319
2320
2321
2322
2323 eb.request->batch = eb.batch;
2324
2325 trace_i915_gem_request_queue(eb.request, eb.batch_flags);
2326 err = eb_submit(&eb);
2327err_request:
2328 __i915_add_request(eb.request, err == 0);
2329 add_to_client(eb.request, file);
2330
2331 if (out_fence) {
2332 if (err == 0) {
2333 fd_install(out_fence_fd, out_fence->file);
2334 args->rsvd2 &= GENMASK_ULL(0, 31);
2335 args->rsvd2 |= (u64)out_fence_fd << 32;
2336 out_fence_fd = -1;
2337 } else {
2338 fput(out_fence->file);
2339 }
2340 }
2341
2342err_batch_unpin:
2343 if (eb.batch_flags & I915_DISPATCH_SECURE)
2344 i915_vma_unpin(eb.batch);
2345err_vma:
2346 if (eb.exec)
2347 eb_release_vmas(&eb);
2348 i915_gem_context_put(eb.ctx);
2349err_unlock:
2350 mutex_unlock(&dev->struct_mutex);
2351err_rpm:
2352 intel_runtime_pm_put(eb.i915);
2353 eb_destroy(&eb);
2354err_out_fence:
2355 if (out_fence_fd != -1)
2356 put_unused_fd(out_fence_fd);
2357err_in_fence:
2358 dma_fence_put(in_fence);
2359 return err;
2360}
2361
2362
2363
2364
2365
2366int
2367i915_gem_execbuffer(struct drm_device *dev, void *data,
2368 struct drm_file *file)
2369{
2370 const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
2371 struct drm_i915_gem_execbuffer *args = data;
2372 struct drm_i915_gem_execbuffer2 exec2;
2373 struct drm_i915_gem_exec_object *exec_list = NULL;
2374 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
2375 unsigned int i;
2376 int err;
2377
2378 if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
2379 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
2380 return -EINVAL;
2381 }
2382
2383 exec2.buffers_ptr = args->buffers_ptr;
2384 exec2.buffer_count = args->buffer_count;
2385 exec2.batch_start_offset = args->batch_start_offset;
2386 exec2.batch_len = args->batch_len;
2387 exec2.DR1 = args->DR1;
2388 exec2.DR4 = args->DR4;
2389 exec2.num_cliprects = args->num_cliprects;
2390 exec2.cliprects_ptr = args->cliprects_ptr;
2391 exec2.flags = I915_EXEC_RENDER;
2392 i915_execbuffer2_set_context_id(exec2, 0);
2393
2394 if (!i915_gem_check_execbuffer(&exec2))
2395 return -EINVAL;
2396
2397
2398 exec_list = kvmalloc_array(args->buffer_count, sizeof(*exec_list),
2399 __GFP_NOWARN | GFP_TEMPORARY);
2400 exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
2401 __GFP_NOWARN | GFP_TEMPORARY);
2402 if (exec_list == NULL || exec2_list == NULL) {
2403 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
2404 args->buffer_count);
2405 kvfree(exec_list);
2406 kvfree(exec2_list);
2407 return -ENOMEM;
2408 }
2409 err = copy_from_user(exec_list,
2410 u64_to_user_ptr(args->buffers_ptr),
2411 sizeof(*exec_list) * args->buffer_count);
2412 if (err) {
2413 DRM_DEBUG("copy %d exec entries failed %d\n",
2414 args->buffer_count, err);
2415 kvfree(exec_list);
2416 kvfree(exec2_list);
2417 return -EFAULT;
2418 }
2419
2420 for (i = 0; i < args->buffer_count; i++) {
2421 exec2_list[i].handle = exec_list[i].handle;
2422 exec2_list[i].relocation_count = exec_list[i].relocation_count;
2423 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
2424 exec2_list[i].alignment = exec_list[i].alignment;
2425 exec2_list[i].offset = exec_list[i].offset;
2426 if (INTEL_GEN(to_i915(dev)) < 4)
2427 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
2428 else
2429 exec2_list[i].flags = 0;
2430 }
2431
2432 err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
2433 if (exec2.flags & __EXEC_HAS_RELOC) {
2434 struct drm_i915_gem_exec_object __user *user_exec_list =
2435 u64_to_user_ptr(args->buffers_ptr);
2436
2437
2438 for (i = 0; i < args->buffer_count; i++) {
2439 if (!(exec2_list[i].offset & UPDATE))
2440 continue;
2441
2442 exec2_list[i].offset =
2443 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
2444 exec2_list[i].offset &= PIN_OFFSET_MASK;
2445 if (__copy_to_user(&user_exec_list[i].offset,
2446 &exec2_list[i].offset,
2447 sizeof(user_exec_list[i].offset)))
2448 break;
2449 }
2450 }
2451
2452 kvfree(exec_list);
2453 kvfree(exec2_list);
2454 return err;
2455}
2456
2457int
2458i915_gem_execbuffer2(struct drm_device *dev, void *data,
2459 struct drm_file *file)
2460{
2461 const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
2462 struct drm_i915_gem_execbuffer2 *args = data;
2463 struct drm_i915_gem_exec_object2 *exec2_list;
2464 int err;
2465
2466 if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
2467 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
2468 return -EINVAL;
2469 }
2470
2471 if (!i915_gem_check_execbuffer(args))
2472 return -EINVAL;
2473
2474
2475 exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
2476 __GFP_NOWARN | GFP_TEMPORARY);
2477 if (exec2_list == NULL) {
2478 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
2479 args->buffer_count);
2480 return -ENOMEM;
2481 }
2482 if (copy_from_user(exec2_list,
2483 u64_to_user_ptr(args->buffers_ptr),
2484 sizeof(*exec2_list) * args->buffer_count)) {
2485 DRM_DEBUG("copy %d exec entries failed\n", args->buffer_count);
2486 kvfree(exec2_list);
2487 return -EFAULT;
2488 }
2489
2490 err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
2491
2492
2493
2494
2495
2496
2497
2498 if (args->flags & __EXEC_HAS_RELOC) {
2499 struct drm_i915_gem_exec_object2 __user *user_exec_list =
2500 u64_to_user_ptr(args->buffers_ptr);
2501 unsigned int i;
2502
2503
2504 user_access_begin();
2505 for (i = 0; i < args->buffer_count; i++) {
2506 if (!(exec2_list[i].offset & UPDATE))
2507 continue;
2508
2509 exec2_list[i].offset =
2510 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
2511 unsafe_put_user(exec2_list[i].offset,
2512 &user_exec_list[i].offset,
2513 end_user);
2514 }
2515end_user:
2516 user_access_end();
2517 }
2518
2519 args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
2520 kvfree(exec2_list);
2521 return err;
2522}
2523