1
2
3
4
5
6
7#include <linux/intel-iommu.h>
8#include <linux/dma-resv.h>
9#include <linux/sync_file.h>
10#include <linux/uaccess.h>
11
12#include <drm/drm_syncobj.h>
13
14#include "display/intel_frontbuffer.h"
15
16#include "gem/i915_gem_ioctls.h"
17#include "gt/intel_context.h"
18#include "gt/intel_gpu_commands.h"
19#include "gt/intel_gt.h"
20#include "gt/intel_gt_buffer_pool.h"
21#include "gt/intel_gt_pm.h"
22#include "gt/intel_ring.h"
23
24#include "pxp/intel_pxp.h"
25
26#include "i915_drv.h"
27#include "i915_gem_clflush.h"
28#include "i915_gem_context.h"
29#include "i915_gem_ioctls.h"
30#include "i915_trace.h"
31#include "i915_user_extensions.h"
32
33struct eb_vma {
34 struct i915_vma *vma;
35 unsigned int flags;
36
37
38 struct drm_i915_gem_exec_object2 *exec;
39 struct list_head bind_link;
40 struct list_head reloc_link;
41
42 struct hlist_node node;
43 u32 handle;
44};
45
46enum {
47 FORCE_CPU_RELOC = 1,
48 FORCE_GTT_RELOC,
49 FORCE_GPU_RELOC,
50#define DBG_FORCE_RELOC 0
51};
52
53
54#define __EXEC_OBJECT_HAS_PIN BIT(30)
55#define __EXEC_OBJECT_HAS_FENCE BIT(29)
56#define __EXEC_OBJECT_USERPTR_INIT BIT(28)
57#define __EXEC_OBJECT_NEEDS_MAP BIT(27)
58#define __EXEC_OBJECT_NEEDS_BIAS BIT(26)
59#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 26)
60#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
61
62#define __EXEC_HAS_RELOC BIT(31)
63#define __EXEC_ENGINE_PINNED BIT(30)
64#define __EXEC_USERPTR_USED BIT(29)
65#define __EXEC_INTERNAL_FLAGS (~0u << 29)
66#define UPDATE PIN_OFFSET_FIXED
67
68#define BATCH_OFFSET_BIAS (256*1024)
69
70#define __I915_EXEC_ILLEGAL_FLAGS \
71 (__I915_EXEC_UNKNOWN_FLAGS | \
72 I915_EXEC_CONSTANTS_MASK | \
73 I915_EXEC_RESOURCE_STREAMER)
74
75
76#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
77#undef EINVAL
78#define EINVAL ({ \
79 DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \
80 22; \
81})
82#endif
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235struct eb_fence {
236 struct drm_syncobj *syncobj;
237 struct dma_fence *dma_fence;
238 u64 value;
239 struct dma_fence_chain *chain_fence;
240};
241
242struct i915_execbuffer {
243 struct drm_i915_private *i915;
244 struct drm_file *file;
245 struct drm_i915_gem_execbuffer2 *args;
246 struct drm_i915_gem_exec_object2 *exec;
247 struct eb_vma *vma;
248
249 struct intel_gt *gt;
250 struct intel_context *context;
251 struct i915_gem_context *gem_context;
252
253
254 struct i915_request *requests[MAX_ENGINE_INSTANCE + 1];
255
256 struct eb_vma *batches[MAX_ENGINE_INSTANCE + 1];
257 struct i915_vma *trampoline;
258
259
260 struct dma_fence *composite_fence;
261
262
263 unsigned int buffer_count;
264
265
266 unsigned int num_batches;
267
268
269 struct list_head unbound;
270
271
272 struct list_head relocs;
273
274 struct i915_gem_ww_ctx ww;
275
276
277
278
279
280
281 struct reloc_cache {
282 struct drm_mm_node node;
283 unsigned long vaddr;
284 unsigned long page;
285 unsigned int graphics_ver;
286 bool use_64bit_reloc : 1;
287 bool has_llc : 1;
288 bool has_fence : 1;
289 bool needs_unfenced : 1;
290 } reloc_cache;
291
292 u64 invalid_flags;
293
294
295 u64 batch_len[MAX_ENGINE_INSTANCE + 1];
296 u32 batch_start_offset;
297 u32 batch_flags;
298 struct intel_gt_buffer_pool_node *batch_pool;
299
300
301
302
303
304
305 int lut_size;
306 struct hlist_head *buckets;
307
308 struct eb_fence *fences;
309 unsigned long num_fences;
310};
311
312static int eb_parse(struct i915_execbuffer *eb);
313static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle);
314static void eb_unpin_engine(struct i915_execbuffer *eb);
315
316static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
317{
318 return intel_engine_requires_cmd_parser(eb->context->engine) ||
319 (intel_engine_using_cmd_parser(eb->context->engine) &&
320 eb->args->batch_len);
321}
322
323static int eb_create(struct i915_execbuffer *eb)
324{
325 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
326 unsigned int size = 1 + ilog2(eb->buffer_count);
327
328
329
330
331
332
333
334
335
336
337
338
339 do {
340 gfp_t flags;
341
342
343
344
345
346
347
348 flags = GFP_KERNEL;
349 if (size > 1)
350 flags |= __GFP_NORETRY | __GFP_NOWARN;
351
352 eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
353 flags);
354 if (eb->buckets)
355 break;
356 } while (--size);
357
358 if (unlikely(!size))
359 return -ENOMEM;
360
361 eb->lut_size = size;
362 } else {
363 eb->lut_size = -eb->buffer_count;
364 }
365
366 return 0;
367}
368
369static bool
370eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
371 const struct i915_vma *vma,
372 unsigned int flags)
373{
374 if (vma->node.size < entry->pad_to_size)
375 return true;
376
377 if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
378 return true;
379
380 if (flags & EXEC_OBJECT_PINNED &&
381 vma->node.start != entry->offset)
382 return true;
383
384 if (flags & __EXEC_OBJECT_NEEDS_BIAS &&
385 vma->node.start < BATCH_OFFSET_BIAS)
386 return true;
387
388 if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
389 (vma->node.start + vma->node.size + 4095) >> 32)
390 return true;
391
392 if (flags & __EXEC_OBJECT_NEEDS_MAP &&
393 !i915_vma_is_map_and_fenceable(vma))
394 return true;
395
396 return false;
397}
398
399static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry,
400 unsigned int exec_flags)
401{
402 u64 pin_flags = 0;
403
404 if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
405 pin_flags |= PIN_GLOBAL;
406
407
408
409
410
411 if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
412 pin_flags |= PIN_ZONE_4G;
413
414 if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
415 pin_flags |= PIN_MAPPABLE;
416
417 if (exec_flags & EXEC_OBJECT_PINNED)
418 pin_flags |= entry->offset | PIN_OFFSET_FIXED;
419 else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS)
420 pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
421
422 return pin_flags;
423}
424
425static inline int
426eb_pin_vma(struct i915_execbuffer *eb,
427 const struct drm_i915_gem_exec_object2 *entry,
428 struct eb_vma *ev)
429{
430 struct i915_vma *vma = ev->vma;
431 u64 pin_flags;
432 int err;
433
434 if (vma->node.size)
435 pin_flags = vma->node.start;
436 else
437 pin_flags = entry->offset & PIN_OFFSET_MASK;
438
439 pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
440 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT))
441 pin_flags |= PIN_GLOBAL;
442
443
444 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags);
445 if (err == -EDEADLK)
446 return err;
447
448 if (unlikely(err)) {
449 if (entry->flags & EXEC_OBJECT_PINNED)
450 return err;
451
452
453 err = i915_vma_pin_ww(vma, &eb->ww,
454 entry->pad_to_size,
455 entry->alignment,
456 eb_pin_flags(entry, ev->flags) |
457 PIN_USER | PIN_NOEVICT);
458 if (unlikely(err))
459 return err;
460 }
461
462 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
463 err = i915_vma_pin_fence(vma);
464 if (unlikely(err)) {
465 i915_vma_unpin(vma);
466 return err;
467 }
468
469 if (vma->fence)
470 ev->flags |= __EXEC_OBJECT_HAS_FENCE;
471 }
472
473 ev->flags |= __EXEC_OBJECT_HAS_PIN;
474 if (eb_vma_misplaced(entry, vma, ev->flags))
475 return -EBADSLT;
476
477 return 0;
478}
479
480static inline void
481eb_unreserve_vma(struct eb_vma *ev)
482{
483 if (!(ev->flags & __EXEC_OBJECT_HAS_PIN))
484 return;
485
486 if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE))
487 __i915_vma_unpin_fence(ev->vma);
488
489 __i915_vma_unpin(ev->vma);
490 ev->flags &= ~__EXEC_OBJECT_RESERVED;
491}
492
493static int
494eb_validate_vma(struct i915_execbuffer *eb,
495 struct drm_i915_gem_exec_object2 *entry,
496 struct i915_vma *vma)
497{
498
499
500
501 if (entry->relocation_count &&
502 GRAPHICS_VER(eb->i915) >= 12 && !IS_TIGERLAKE(eb->i915))
503 return -EINVAL;
504
505 if (unlikely(entry->flags & eb->invalid_flags))
506 return -EINVAL;
507
508 if (unlikely(entry->alignment &&
509 !is_power_of_2_u64(entry->alignment)))
510 return -EINVAL;
511
512
513
514
515
516 if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
517 entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK)))
518 return -EINVAL;
519
520
521 if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
522 if (unlikely(offset_in_page(entry->pad_to_size)))
523 return -EINVAL;
524 } else {
525 entry->pad_to_size = 0;
526 }
527
528
529
530
531
532 entry->offset = gen8_noncanonical_addr(entry->offset);
533
534 if (!eb->reloc_cache.has_fence) {
535 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
536 } else {
537 if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
538 eb->reloc_cache.needs_unfenced) &&
539 i915_gem_object_is_tiled(vma->obj))
540 entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
541 }
542
543 return 0;
544}
545
546static inline bool
547is_batch_buffer(struct i915_execbuffer *eb, unsigned int buffer_idx)
548{
549 return eb->args->flags & I915_EXEC_BATCH_FIRST ?
550 buffer_idx < eb->num_batches :
551 buffer_idx >= eb->args->buffer_count - eb->num_batches;
552}
553
554static int
555eb_add_vma(struct i915_execbuffer *eb,
556 unsigned int *current_batch,
557 unsigned int i,
558 struct i915_vma *vma)
559{
560 struct drm_i915_private *i915 = eb->i915;
561 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
562 struct eb_vma *ev = &eb->vma[i];
563
564 ev->vma = vma;
565 ev->exec = entry;
566 ev->flags = entry->flags;
567
568 if (eb->lut_size > 0) {
569 ev->handle = entry->handle;
570 hlist_add_head(&ev->node,
571 &eb->buckets[hash_32(entry->handle,
572 eb->lut_size)]);
573 }
574
575 if (entry->relocation_count)
576 list_add_tail(&ev->reloc_link, &eb->relocs);
577
578
579
580
581
582
583
584
585
586
587 if (is_batch_buffer(eb, i)) {
588 if (entry->relocation_count &&
589 !(ev->flags & EXEC_OBJECT_PINNED))
590 ev->flags |= __EXEC_OBJECT_NEEDS_BIAS;
591 if (eb->reloc_cache.has_fence)
592 ev->flags |= EXEC_OBJECT_NEEDS_FENCE;
593
594 eb->batches[*current_batch] = ev;
595
596 if (unlikely(ev->flags & EXEC_OBJECT_WRITE)) {
597 drm_dbg(&i915->drm,
598 "Attempting to use self-modifying batch buffer\n");
599 return -EINVAL;
600 }
601
602 if (range_overflows_t(u64,
603 eb->batch_start_offset,
604 eb->args->batch_len,
605 ev->vma->size)) {
606 drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n");
607 return -EINVAL;
608 }
609
610 if (eb->args->batch_len == 0)
611 eb->batch_len[*current_batch] = ev->vma->size -
612 eb->batch_start_offset;
613 else
614 eb->batch_len[*current_batch] = eb->args->batch_len;
615 if (unlikely(eb->batch_len[*current_batch] == 0)) {
616 drm_dbg(&i915->drm, "Invalid batch length\n");
617 return -EINVAL;
618 }
619
620 ++*current_batch;
621 }
622
623 return 0;
624}
625
626static inline int use_cpu_reloc(const struct reloc_cache *cache,
627 const struct drm_i915_gem_object *obj)
628{
629 if (!i915_gem_object_has_struct_page(obj))
630 return false;
631
632 if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
633 return true;
634
635 if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
636 return false;
637
638 return (cache->has_llc ||
639 obj->cache_dirty ||
640 obj->cache_level != I915_CACHE_NONE);
641}
642
643static int eb_reserve_vma(struct i915_execbuffer *eb,
644 struct eb_vma *ev,
645 u64 pin_flags)
646{
647 struct drm_i915_gem_exec_object2 *entry = ev->exec;
648 struct i915_vma *vma = ev->vma;
649 int err;
650
651 if (drm_mm_node_allocated(&vma->node) &&
652 eb_vma_misplaced(entry, vma, ev->flags)) {
653 err = i915_vma_unbind(vma);
654 if (err)
655 return err;
656 }
657
658 err = i915_vma_pin_ww(vma, &eb->ww,
659 entry->pad_to_size, entry->alignment,
660 eb_pin_flags(entry, ev->flags) | pin_flags);
661 if (err)
662 return err;
663
664 if (entry->offset != vma->node.start) {
665 entry->offset = vma->node.start | UPDATE;
666 eb->args->flags |= __EXEC_HAS_RELOC;
667 }
668
669 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
670 err = i915_vma_pin_fence(vma);
671 if (unlikely(err)) {
672 i915_vma_unpin(vma);
673 return err;
674 }
675
676 if (vma->fence)
677 ev->flags |= __EXEC_OBJECT_HAS_FENCE;
678 }
679
680 ev->flags |= __EXEC_OBJECT_HAS_PIN;
681 GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags));
682
683 return 0;
684}
685
686static int eb_reserve(struct i915_execbuffer *eb)
687{
688 const unsigned int count = eb->buffer_count;
689 unsigned int pin_flags = PIN_USER | PIN_NONBLOCK;
690 struct list_head last;
691 struct eb_vma *ev;
692 unsigned int i, pass;
693 int err = 0;
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708 pass = 0;
709 do {
710 list_for_each_entry(ev, &eb->unbound, bind_link) {
711 err = eb_reserve_vma(eb, ev, pin_flags);
712 if (err)
713 break;
714 }
715 if (err != -ENOSPC)
716 return err;
717
718
719 INIT_LIST_HEAD(&eb->unbound);
720 INIT_LIST_HEAD(&last);
721 for (i = 0; i < count; i++) {
722 unsigned int flags;
723
724 ev = &eb->vma[i];
725 flags = ev->flags;
726 if (flags & EXEC_OBJECT_PINNED &&
727 flags & __EXEC_OBJECT_HAS_PIN)
728 continue;
729
730 eb_unreserve_vma(ev);
731
732 if (flags & EXEC_OBJECT_PINNED)
733
734 list_add(&ev->bind_link, &eb->unbound);
735 else if (flags & __EXEC_OBJECT_NEEDS_MAP)
736
737 list_add_tail(&ev->bind_link, &eb->unbound);
738 else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
739
740 list_add(&ev->bind_link, &last);
741 else
742 list_add_tail(&ev->bind_link, &last);
743 }
744 list_splice_tail(&last, &eb->unbound);
745
746 switch (pass++) {
747 case 0:
748 break;
749
750 case 1:
751
752 mutex_lock(&eb->context->vm->mutex);
753 err = i915_gem_evict_vm(eb->context->vm);
754 mutex_unlock(&eb->context->vm->mutex);
755 if (err)
756 return err;
757 break;
758
759 default:
760 return -ENOSPC;
761 }
762
763 pin_flags = PIN_USER;
764 } while (1);
765}
766
767static int eb_select_context(struct i915_execbuffer *eb)
768{
769 struct i915_gem_context *ctx;
770
771 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
772 if (unlikely(IS_ERR(ctx)))
773 return PTR_ERR(ctx);
774
775 eb->gem_context = ctx;
776 if (i915_gem_context_has_full_ppgtt(ctx))
777 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
778
779 return 0;
780}
781
782static int __eb_add_lut(struct i915_execbuffer *eb,
783 u32 handle, struct i915_vma *vma)
784{
785 struct i915_gem_context *ctx = eb->gem_context;
786 struct i915_lut_handle *lut;
787 int err;
788
789 lut = i915_lut_handle_alloc();
790 if (unlikely(!lut))
791 return -ENOMEM;
792
793 i915_vma_get(vma);
794 if (!atomic_fetch_inc(&vma->open_count))
795 i915_vma_reopen(vma);
796 lut->handle = handle;
797 lut->ctx = ctx;
798
799
800 err = -EINTR;
801 if (!mutex_lock_interruptible(&ctx->lut_mutex)) {
802 if (likely(!i915_gem_context_is_closed(ctx)))
803 err = radix_tree_insert(&ctx->handles_vma, handle, vma);
804 else
805 err = -ENOENT;
806 if (err == 0) {
807 struct drm_i915_gem_object *obj = vma->obj;
808
809 spin_lock(&obj->lut_lock);
810 if (idr_find(&eb->file->object_idr, handle) == obj) {
811 list_add(&lut->obj_link, &obj->lut_list);
812 } else {
813 radix_tree_delete(&ctx->handles_vma, handle);
814 err = -ENOENT;
815 }
816 spin_unlock(&obj->lut_lock);
817 }
818 mutex_unlock(&ctx->lut_mutex);
819 }
820 if (unlikely(err))
821 goto err;
822
823 return 0;
824
825err:
826 i915_vma_close(vma);
827 i915_vma_put(vma);
828 i915_lut_handle_free(lut);
829 return err;
830}
831
832static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle)
833{
834 struct i915_address_space *vm = eb->context->vm;
835
836 do {
837 struct drm_i915_gem_object *obj;
838 struct i915_vma *vma;
839 int err;
840
841 rcu_read_lock();
842 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle);
843 if (likely(vma && vma->vm == vm))
844 vma = i915_vma_tryget(vma);
845 rcu_read_unlock();
846 if (likely(vma))
847 return vma;
848
849 obj = i915_gem_object_lookup(eb->file, handle);
850 if (unlikely(!obj))
851 return ERR_PTR(-ENOENT);
852
853
854
855
856
857
858
859
860 if (i915_gem_context_uses_protected_content(eb->gem_context) &&
861 i915_gem_object_is_protected(obj)) {
862 err = intel_pxp_key_check(&vm->gt->pxp, obj, true);
863 if (err) {
864 i915_gem_object_put(obj);
865 return ERR_PTR(err);
866 }
867 }
868
869 vma = i915_vma_instance(obj, vm, NULL);
870 if (IS_ERR(vma)) {
871 i915_gem_object_put(obj);
872 return vma;
873 }
874
875 err = __eb_add_lut(eb, handle, vma);
876 if (likely(!err))
877 return vma;
878
879 i915_gem_object_put(obj);
880 if (err != -EEXIST)
881 return ERR_PTR(err);
882 } while (1);
883}
884
885static int eb_lookup_vmas(struct i915_execbuffer *eb)
886{
887 unsigned int i, current_batch = 0;
888 int err = 0;
889
890 INIT_LIST_HEAD(&eb->relocs);
891
892 for (i = 0; i < eb->buffer_count; i++) {
893 struct i915_vma *vma;
894
895 vma = eb_lookup_vma(eb, eb->exec[i].handle);
896 if (IS_ERR(vma)) {
897 err = PTR_ERR(vma);
898 goto err;
899 }
900
901 err = eb_validate_vma(eb, &eb->exec[i], vma);
902 if (unlikely(err)) {
903 i915_vma_put(vma);
904 goto err;
905 }
906
907 err = eb_add_vma(eb, ¤t_batch, i, vma);
908 if (err)
909 return err;
910
911 if (i915_gem_object_is_userptr(vma->obj)) {
912 err = i915_gem_object_userptr_submit_init(vma->obj);
913 if (err) {
914 if (i + 1 < eb->buffer_count) {
915
916
917
918
919
920
921 eb->vma[i + 1].vma = NULL;
922 }
923
924 return err;
925 }
926
927 eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT;
928 eb->args->flags |= __EXEC_USERPTR_USED;
929 }
930 }
931
932 return 0;
933
934err:
935 eb->vma[i].vma = NULL;
936 return err;
937}
938
939static int eb_lock_vmas(struct i915_execbuffer *eb)
940{
941 unsigned int i;
942 int err;
943
944 for (i = 0; i < eb->buffer_count; i++) {
945 struct eb_vma *ev = &eb->vma[i];
946 struct i915_vma *vma = ev->vma;
947
948 err = i915_gem_object_lock(vma->obj, &eb->ww);
949 if (err)
950 return err;
951 }
952
953 return 0;
954}
955
956static int eb_validate_vmas(struct i915_execbuffer *eb)
957{
958 unsigned int i;
959 int err;
960
961 INIT_LIST_HEAD(&eb->unbound);
962
963 err = eb_lock_vmas(eb);
964 if (err)
965 return err;
966
967 for (i = 0; i < eb->buffer_count; i++) {
968 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
969 struct eb_vma *ev = &eb->vma[i];
970 struct i915_vma *vma = ev->vma;
971
972 err = eb_pin_vma(eb, entry, ev);
973 if (err == -EDEADLK)
974 return err;
975
976 if (!err) {
977 if (entry->offset != vma->node.start) {
978 entry->offset = vma->node.start | UPDATE;
979 eb->args->flags |= __EXEC_HAS_RELOC;
980 }
981 } else {
982 eb_unreserve_vma(ev);
983
984 list_add_tail(&ev->bind_link, &eb->unbound);
985 if (drm_mm_node_allocated(&vma->node)) {
986 err = i915_vma_unbind(vma);
987 if (err)
988 return err;
989 }
990 }
991
992 if (!(ev->flags & EXEC_OBJECT_WRITE)) {
993 err = dma_resv_reserve_shared(vma->resv, 1);
994 if (err)
995 return err;
996 }
997
998 GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
999 eb_vma_misplaced(&eb->exec[i], vma, ev->flags));
1000 }
1001
1002 if (!list_empty(&eb->unbound))
1003 return eb_reserve(eb);
1004
1005 return 0;
1006}
1007
1008static struct eb_vma *
1009eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
1010{
1011 if (eb->lut_size < 0) {
1012 if (handle >= -eb->lut_size)
1013 return NULL;
1014 return &eb->vma[handle];
1015 } else {
1016 struct hlist_head *head;
1017 struct eb_vma *ev;
1018
1019 head = &eb->buckets[hash_32(handle, eb->lut_size)];
1020 hlist_for_each_entry(ev, head, node) {
1021 if (ev->handle == handle)
1022 return ev;
1023 }
1024 return NULL;
1025 }
1026}
1027
1028static void eb_release_vmas(struct i915_execbuffer *eb, bool final)
1029{
1030 const unsigned int count = eb->buffer_count;
1031 unsigned int i;
1032
1033 for (i = 0; i < count; i++) {
1034 struct eb_vma *ev = &eb->vma[i];
1035 struct i915_vma *vma = ev->vma;
1036
1037 if (!vma)
1038 break;
1039
1040 eb_unreserve_vma(ev);
1041
1042 if (final)
1043 i915_vma_put(vma);
1044 }
1045
1046 eb_unpin_engine(eb);
1047}
1048
1049static void eb_destroy(const struct i915_execbuffer *eb)
1050{
1051 if (eb->lut_size > 0)
1052 kfree(eb->buckets);
1053}
1054
1055static inline u64
1056relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
1057 const struct i915_vma *target)
1058{
1059 return gen8_canonical_addr((int)reloc->delta + target->node.start);
1060}
1061
1062static void reloc_cache_init(struct reloc_cache *cache,
1063 struct drm_i915_private *i915)
1064{
1065 cache->page = -1;
1066 cache->vaddr = 0;
1067
1068 cache->graphics_ver = GRAPHICS_VER(i915);
1069 cache->has_llc = HAS_LLC(i915);
1070 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
1071 cache->has_fence = cache->graphics_ver < 4;
1072 cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
1073 cache->node.flags = 0;
1074}
1075
1076static inline void *unmask_page(unsigned long p)
1077{
1078 return (void *)(uintptr_t)(p & PAGE_MASK);
1079}
1080
1081static inline unsigned int unmask_flags(unsigned long p)
1082{
1083 return p & ~PAGE_MASK;
1084}
1085
1086#define KMAP 0x4
1087
1088static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
1089{
1090 struct drm_i915_private *i915 =
1091 container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
1092 return &i915->ggtt;
1093}
1094
1095static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb)
1096{
1097 void *vaddr;
1098
1099 if (!cache->vaddr)
1100 return;
1101
1102 vaddr = unmask_page(cache->vaddr);
1103 if (cache->vaddr & KMAP) {
1104 struct drm_i915_gem_object *obj =
1105 (struct drm_i915_gem_object *)cache->node.mm;
1106 if (cache->vaddr & CLFLUSH_AFTER)
1107 mb();
1108
1109 kunmap_atomic(vaddr);
1110 i915_gem_object_finish_access(obj);
1111 } else {
1112 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1113
1114 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1115 io_mapping_unmap_atomic((void __iomem *)vaddr);
1116
1117 if (drm_mm_node_allocated(&cache->node)) {
1118 ggtt->vm.clear_range(&ggtt->vm,
1119 cache->node.start,
1120 cache->node.size);
1121 mutex_lock(&ggtt->vm.mutex);
1122 drm_mm_remove_node(&cache->node);
1123 mutex_unlock(&ggtt->vm.mutex);
1124 } else {
1125 i915_vma_unpin((struct i915_vma *)cache->node.mm);
1126 }
1127 }
1128
1129 cache->vaddr = 0;
1130 cache->page = -1;
1131}
1132
1133static void *reloc_kmap(struct drm_i915_gem_object *obj,
1134 struct reloc_cache *cache,
1135 unsigned long pageno)
1136{
1137 void *vaddr;
1138 struct page *page;
1139
1140 if (cache->vaddr) {
1141 kunmap_atomic(unmask_page(cache->vaddr));
1142 } else {
1143 unsigned int flushes;
1144 int err;
1145
1146 err = i915_gem_object_prepare_write(obj, &flushes);
1147 if (err)
1148 return ERR_PTR(err);
1149
1150 BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
1151 BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
1152
1153 cache->vaddr = flushes | KMAP;
1154 cache->node.mm = (void *)obj;
1155 if (flushes)
1156 mb();
1157 }
1158
1159 page = i915_gem_object_get_page(obj, pageno);
1160 if (!obj->mm.dirty)
1161 set_page_dirty(page);
1162
1163 vaddr = kmap_atomic(page);
1164 cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
1165 cache->page = pageno;
1166
1167 return vaddr;
1168}
1169
1170static void *reloc_iomap(struct drm_i915_gem_object *obj,
1171 struct i915_execbuffer *eb,
1172 unsigned long page)
1173{
1174 struct reloc_cache *cache = &eb->reloc_cache;
1175 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1176 unsigned long offset;
1177 void *vaddr;
1178
1179 if (cache->vaddr) {
1180 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1181 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
1182 } else {
1183 struct i915_vma *vma;
1184 int err;
1185
1186 if (i915_gem_object_is_tiled(obj))
1187 return ERR_PTR(-EINVAL);
1188
1189 if (use_cpu_reloc(cache, obj))
1190 return NULL;
1191
1192 err = i915_gem_object_set_to_gtt_domain(obj, true);
1193 if (err)
1194 return ERR_PTR(err);
1195
1196 vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0,
1197 PIN_MAPPABLE |
1198 PIN_NONBLOCK |
1199 PIN_NOEVICT);
1200 if (vma == ERR_PTR(-EDEADLK))
1201 return vma;
1202
1203 if (IS_ERR(vma)) {
1204 memset(&cache->node, 0, sizeof(cache->node));
1205 mutex_lock(&ggtt->vm.mutex);
1206 err = drm_mm_insert_node_in_range
1207 (&ggtt->vm.mm, &cache->node,
1208 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
1209 0, ggtt->mappable_end,
1210 DRM_MM_INSERT_LOW);
1211 mutex_unlock(&ggtt->vm.mutex);
1212 if (err)
1213 return NULL;
1214 } else {
1215 cache->node.start = vma->node.start;
1216 cache->node.mm = (void *)vma;
1217 }
1218 }
1219
1220 offset = cache->node.start;
1221 if (drm_mm_node_allocated(&cache->node)) {
1222 ggtt->vm.insert_page(&ggtt->vm,
1223 i915_gem_object_get_dma_address(obj, page),
1224 offset, I915_CACHE_NONE, 0);
1225 } else {
1226 offset += page << PAGE_SHIFT;
1227 }
1228
1229 vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap,
1230 offset);
1231 cache->page = page;
1232 cache->vaddr = (unsigned long)vaddr;
1233
1234 return vaddr;
1235}
1236
1237static void *reloc_vaddr(struct drm_i915_gem_object *obj,
1238 struct i915_execbuffer *eb,
1239 unsigned long page)
1240{
1241 struct reloc_cache *cache = &eb->reloc_cache;
1242 void *vaddr;
1243
1244 if (cache->page == page) {
1245 vaddr = unmask_page(cache->vaddr);
1246 } else {
1247 vaddr = NULL;
1248 if ((cache->vaddr & KMAP) == 0)
1249 vaddr = reloc_iomap(obj, eb, page);
1250 if (!vaddr)
1251 vaddr = reloc_kmap(obj, cache, page);
1252 }
1253
1254 return vaddr;
1255}
1256
1257static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
1258{
1259 if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
1260 if (flushes & CLFLUSH_BEFORE) {
1261 clflushopt(addr);
1262 mb();
1263 }
1264
1265 *addr = value;
1266
1267
1268
1269
1270
1271
1272
1273
1274 if (flushes & CLFLUSH_AFTER)
1275 clflushopt(addr);
1276 } else
1277 *addr = value;
1278}
1279
1280static u64
1281relocate_entry(struct i915_vma *vma,
1282 const struct drm_i915_gem_relocation_entry *reloc,
1283 struct i915_execbuffer *eb,
1284 const struct i915_vma *target)
1285{
1286 u64 target_addr = relocation_target(reloc, target);
1287 u64 offset = reloc->offset;
1288 bool wide = eb->reloc_cache.use_64bit_reloc;
1289 void *vaddr;
1290
1291repeat:
1292 vaddr = reloc_vaddr(vma->obj, eb,
1293 offset >> PAGE_SHIFT);
1294 if (IS_ERR(vaddr))
1295 return PTR_ERR(vaddr);
1296
1297 GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32)));
1298 clflush_write32(vaddr + offset_in_page(offset),
1299 lower_32_bits(target_addr),
1300 eb->reloc_cache.vaddr);
1301
1302 if (wide) {
1303 offset += sizeof(u32);
1304 target_addr >>= 32;
1305 wide = false;
1306 goto repeat;
1307 }
1308
1309 return target->node.start | UPDATE;
1310}
1311
1312static u64
1313eb_relocate_entry(struct i915_execbuffer *eb,
1314 struct eb_vma *ev,
1315 const struct drm_i915_gem_relocation_entry *reloc)
1316{
1317 struct drm_i915_private *i915 = eb->i915;
1318 struct eb_vma *target;
1319 int err;
1320
1321
1322 target = eb_get_vma(eb, reloc->target_handle);
1323 if (unlikely(!target))
1324 return -ENOENT;
1325
1326
1327 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
1328 drm_dbg(&i915->drm, "reloc with multiple write domains: "
1329 "target %d offset %d "
1330 "read %08x write %08x",
1331 reloc->target_handle,
1332 (int) reloc->offset,
1333 reloc->read_domains,
1334 reloc->write_domain);
1335 return -EINVAL;
1336 }
1337 if (unlikely((reloc->write_domain | reloc->read_domains)
1338 & ~I915_GEM_GPU_DOMAINS)) {
1339 drm_dbg(&i915->drm, "reloc with read/write non-GPU domains: "
1340 "target %d offset %d "
1341 "read %08x write %08x",
1342 reloc->target_handle,
1343 (int) reloc->offset,
1344 reloc->read_domains,
1345 reloc->write_domain);
1346 return -EINVAL;
1347 }
1348
1349 if (reloc->write_domain) {
1350 target->flags |= EXEC_OBJECT_WRITE;
1351
1352
1353
1354
1355
1356
1357
1358 if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
1359 GRAPHICS_VER(eb->i915) == 6) {
1360 err = i915_vma_bind(target->vma,
1361 target->vma->obj->cache_level,
1362 PIN_GLOBAL, NULL);
1363 if (err)
1364 return err;
1365 }
1366 }
1367
1368
1369
1370
1371
1372 if (!DBG_FORCE_RELOC &&
1373 gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset)
1374 return 0;
1375
1376
1377 if (unlikely(reloc->offset >
1378 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
1379 drm_dbg(&i915->drm, "Relocation beyond object bounds: "
1380 "target %d offset %d size %d.\n",
1381 reloc->target_handle,
1382 (int)reloc->offset,
1383 (int)ev->vma->size);
1384 return -EINVAL;
1385 }
1386 if (unlikely(reloc->offset & 3)) {
1387 drm_dbg(&i915->drm, "Relocation not 4-byte aligned: "
1388 "target %d offset %d.\n",
1389 reloc->target_handle,
1390 (int)reloc->offset);
1391 return -EINVAL;
1392 }
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402 ev->flags &= ~EXEC_OBJECT_ASYNC;
1403
1404
1405 return relocate_entry(ev->vma, reloc, eb, target->vma);
1406}
1407
1408static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev)
1409{
1410#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
1411 struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
1412 const struct drm_i915_gem_exec_object2 *entry = ev->exec;
1413 struct drm_i915_gem_relocation_entry __user *urelocs =
1414 u64_to_user_ptr(entry->relocs_ptr);
1415 unsigned long remain = entry->relocation_count;
1416
1417 if (unlikely(remain > N_RELOC(ULONG_MAX)))
1418 return -EINVAL;
1419
1420
1421
1422
1423
1424
1425 if (unlikely(!access_ok(urelocs, remain * sizeof(*urelocs))))
1426 return -EFAULT;
1427
1428 do {
1429 struct drm_i915_gem_relocation_entry *r = stack;
1430 unsigned int count =
1431 min_t(unsigned long, remain, ARRAY_SIZE(stack));
1432 unsigned int copied;
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442 pagefault_disable();
1443 copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
1444 pagefault_enable();
1445 if (unlikely(copied)) {
1446 remain = -EFAULT;
1447 goto out;
1448 }
1449
1450 remain -= count;
1451 do {
1452 u64 offset = eb_relocate_entry(eb, ev, r);
1453
1454 if (likely(offset == 0)) {
1455 } else if ((s64)offset < 0) {
1456 remain = (int)offset;
1457 goto out;
1458 } else {
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480 offset = gen8_canonical_addr(offset & ~UPDATE);
1481 __put_user(offset,
1482 &urelocs[r - stack].presumed_offset);
1483 }
1484 } while (r++, --count);
1485 urelocs += ARRAY_SIZE(stack);
1486 } while (remain);
1487out:
1488 reloc_cache_reset(&eb->reloc_cache, eb);
1489 return remain;
1490}
1491
1492static int
1493eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev)
1494{
1495 const struct drm_i915_gem_exec_object2 *entry = ev->exec;
1496 struct drm_i915_gem_relocation_entry *relocs =
1497 u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1498 unsigned int i;
1499 int err;
1500
1501 for (i = 0; i < entry->relocation_count; i++) {
1502 u64 offset = eb_relocate_entry(eb, ev, &relocs[i]);
1503
1504 if ((s64)offset < 0) {
1505 err = (int)offset;
1506 goto err;
1507 }
1508 }
1509 err = 0;
1510err:
1511 reloc_cache_reset(&eb->reloc_cache, eb);
1512 return err;
1513}
1514
1515static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
1516{
1517 const char __user *addr, *end;
1518 unsigned long size;
1519 char __maybe_unused c;
1520
1521 size = entry->relocation_count;
1522 if (size == 0)
1523 return 0;
1524
1525 if (size > N_RELOC(ULONG_MAX))
1526 return -EINVAL;
1527
1528 addr = u64_to_user_ptr(entry->relocs_ptr);
1529 size *= sizeof(struct drm_i915_gem_relocation_entry);
1530 if (!access_ok(addr, size))
1531 return -EFAULT;
1532
1533 end = addr + size;
1534 for (; addr < end; addr += PAGE_SIZE) {
1535 int err = __get_user(c, addr);
1536 if (err)
1537 return err;
1538 }
1539 return __get_user(c, end - 1);
1540}
1541
1542static int eb_copy_relocations(const struct i915_execbuffer *eb)
1543{
1544 struct drm_i915_gem_relocation_entry *relocs;
1545 const unsigned int count = eb->buffer_count;
1546 unsigned int i;
1547 int err;
1548
1549 for (i = 0; i < count; i++) {
1550 const unsigned int nreloc = eb->exec[i].relocation_count;
1551 struct drm_i915_gem_relocation_entry __user *urelocs;
1552 unsigned long size;
1553 unsigned long copied;
1554
1555 if (nreloc == 0)
1556 continue;
1557
1558 err = check_relocations(&eb->exec[i]);
1559 if (err)
1560 goto err;
1561
1562 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
1563 size = nreloc * sizeof(*relocs);
1564
1565 relocs = kvmalloc_array(size, 1, GFP_KERNEL);
1566 if (!relocs) {
1567 err = -ENOMEM;
1568 goto err;
1569 }
1570
1571
1572 copied = 0;
1573 do {
1574 unsigned int len =
1575 min_t(u64, BIT_ULL(31), size - copied);
1576
1577 if (__copy_from_user((char *)relocs + copied,
1578 (char __user *)urelocs + copied,
1579 len))
1580 goto end;
1581
1582 copied += len;
1583 } while (copied < size);
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595 if (!user_access_begin(urelocs, size))
1596 goto end;
1597
1598 for (copied = 0; copied < nreloc; copied++)
1599 unsafe_put_user(-1,
1600 &urelocs[copied].presumed_offset,
1601 end_user);
1602 user_access_end();
1603
1604 eb->exec[i].relocs_ptr = (uintptr_t)relocs;
1605 }
1606
1607 return 0;
1608
1609end_user:
1610 user_access_end();
1611end:
1612 kvfree(relocs);
1613 err = -EFAULT;
1614err:
1615 while (i--) {
1616 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
1617 if (eb->exec[i].relocation_count)
1618 kvfree(relocs);
1619 }
1620 return err;
1621}
1622
1623static int eb_prefault_relocations(const struct i915_execbuffer *eb)
1624{
1625 const unsigned int count = eb->buffer_count;
1626 unsigned int i;
1627
1628 for (i = 0; i < count; i++) {
1629 int err;
1630
1631 err = check_relocations(&eb->exec[i]);
1632 if (err)
1633 return err;
1634 }
1635
1636 return 0;
1637}
1638
1639static int eb_reinit_userptr(struct i915_execbuffer *eb)
1640{
1641 const unsigned int count = eb->buffer_count;
1642 unsigned int i;
1643 int ret;
1644
1645 if (likely(!(eb->args->flags & __EXEC_USERPTR_USED)))
1646 return 0;
1647
1648 for (i = 0; i < count; i++) {
1649 struct eb_vma *ev = &eb->vma[i];
1650
1651 if (!i915_gem_object_is_userptr(ev->vma->obj))
1652 continue;
1653
1654 ret = i915_gem_object_userptr_submit_init(ev->vma->obj);
1655 if (ret)
1656 return ret;
1657
1658 ev->flags |= __EXEC_OBJECT_USERPTR_INIT;
1659 }
1660
1661 return 0;
1662}
1663
1664static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb)
1665{
1666 bool have_copy = false;
1667 struct eb_vma *ev;
1668 int err = 0;
1669
1670repeat:
1671 if (signal_pending(current)) {
1672 err = -ERESTARTSYS;
1673 goto out;
1674 }
1675
1676
1677 eb_release_vmas(eb, false);
1678 i915_gem_ww_ctx_fini(&eb->ww);
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693 if (!err) {
1694 err = eb_prefault_relocations(eb);
1695 } else if (!have_copy) {
1696 err = eb_copy_relocations(eb);
1697 have_copy = err == 0;
1698 } else {
1699 cond_resched();
1700 err = 0;
1701 }
1702
1703 if (!err)
1704 err = eb_reinit_userptr(eb);
1705
1706 i915_gem_ww_ctx_init(&eb->ww, true);
1707 if (err)
1708 goto out;
1709
1710
1711repeat_validate:
1712 err = eb_pin_engine(eb, false);
1713 if (err)
1714 goto err;
1715
1716 err = eb_validate_vmas(eb);
1717 if (err)
1718 goto err;
1719
1720 GEM_BUG_ON(!eb->batches[0]);
1721
1722 list_for_each_entry(ev, &eb->relocs, reloc_link) {
1723 if (!have_copy) {
1724 err = eb_relocate_vma(eb, ev);
1725 if (err)
1726 break;
1727 } else {
1728 err = eb_relocate_vma_slow(eb, ev);
1729 if (err)
1730 break;
1731 }
1732 }
1733
1734 if (err == -EDEADLK)
1735 goto err;
1736
1737 if (err && !have_copy)
1738 goto repeat;
1739
1740 if (err)
1741 goto err;
1742
1743
1744 err = eb_parse(eb);
1745 if (err)
1746 goto err;
1747
1748
1749
1750
1751
1752
1753
1754
1755err:
1756 if (err == -EDEADLK) {
1757 eb_release_vmas(eb, false);
1758 err = i915_gem_ww_ctx_backoff(&eb->ww);
1759 if (!err)
1760 goto repeat_validate;
1761 }
1762
1763 if (err == -EAGAIN)
1764 goto repeat;
1765
1766out:
1767 if (have_copy) {
1768 const unsigned int count = eb->buffer_count;
1769 unsigned int i;
1770
1771 for (i = 0; i < count; i++) {
1772 const struct drm_i915_gem_exec_object2 *entry =
1773 &eb->exec[i];
1774 struct drm_i915_gem_relocation_entry *relocs;
1775
1776 if (!entry->relocation_count)
1777 continue;
1778
1779 relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1780 kvfree(relocs);
1781 }
1782 }
1783
1784 return err;
1785}
1786
1787static int eb_relocate_parse(struct i915_execbuffer *eb)
1788{
1789 int err;
1790 bool throttle = true;
1791
1792retry:
1793 err = eb_pin_engine(eb, throttle);
1794 if (err) {
1795 if (err != -EDEADLK)
1796 return err;
1797
1798 goto err;
1799 }
1800
1801
1802 throttle = false;
1803
1804 err = eb_validate_vmas(eb);
1805 if (err == -EAGAIN)
1806 goto slow;
1807 else if (err)
1808 goto err;
1809
1810
1811 if (eb->args->flags & __EXEC_HAS_RELOC) {
1812 struct eb_vma *ev;
1813
1814 list_for_each_entry(ev, &eb->relocs, reloc_link) {
1815 err = eb_relocate_vma(eb, ev);
1816 if (err)
1817 break;
1818 }
1819
1820 if (err == -EDEADLK)
1821 goto err;
1822 else if (err)
1823 goto slow;
1824 }
1825
1826 if (!err)
1827 err = eb_parse(eb);
1828
1829err:
1830 if (err == -EDEADLK) {
1831 eb_release_vmas(eb, false);
1832 err = i915_gem_ww_ctx_backoff(&eb->ww);
1833 if (!err)
1834 goto retry;
1835 }
1836
1837 return err;
1838
1839slow:
1840 err = eb_relocate_parse_slow(eb);
1841 if (err)
1842
1843
1844
1845
1846
1847
1848
1849 eb->args->flags &= ~__EXEC_HAS_RELOC;
1850
1851 return err;
1852}
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863#define for_each_batch_create_order(_eb, _i) \
1864 for ((_i) = 0; (_i) < (_eb)->num_batches; ++(_i))
1865#define for_each_batch_add_order(_eb, _i) \
1866 BUILD_BUG_ON(!typecheck(int, _i)); \
1867 for ((_i) = (_eb)->num_batches - 1; (_i) >= 0; --(_i))
1868
1869static struct i915_request *
1870eb_find_first_request_added(struct i915_execbuffer *eb)
1871{
1872 int i;
1873
1874 for_each_batch_add_order(eb, i)
1875 if (eb->requests[i])
1876 return eb->requests[i];
1877
1878 GEM_BUG_ON("Request not found");
1879
1880 return NULL;
1881}
1882
1883static int eb_move_to_gpu(struct i915_execbuffer *eb)
1884{
1885 const unsigned int count = eb->buffer_count;
1886 unsigned int i = count;
1887 int err = 0, j;
1888
1889 while (i--) {
1890 struct eb_vma *ev = &eb->vma[i];
1891 struct i915_vma *vma = ev->vma;
1892 unsigned int flags = ev->flags;
1893 struct drm_i915_gem_object *obj = vma->obj;
1894
1895 assert_vma_held(vma);
1896
1897 if (flags & EXEC_OBJECT_CAPTURE) {
1898 struct i915_capture_list *capture;
1899
1900 for_each_batch_create_order(eb, j) {
1901 if (!eb->requests[j])
1902 break;
1903
1904 capture = kmalloc(sizeof(*capture), GFP_KERNEL);
1905 if (capture) {
1906 capture->next =
1907 eb->requests[j]->capture_list;
1908 capture->vma = vma;
1909 eb->requests[j]->capture_list = capture;
1910 }
1911 }
1912 }
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937 if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) {
1938 if (i915_gem_clflush_object(obj, 0))
1939 flags &= ~EXEC_OBJECT_ASYNC;
1940 }
1941
1942
1943 if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) {
1944 err = i915_request_await_object
1945 (eb_find_first_request_added(eb), obj,
1946 flags & EXEC_OBJECT_WRITE);
1947 }
1948
1949 for_each_batch_add_order(eb, j) {
1950 if (err)
1951 break;
1952 if (!eb->requests[j])
1953 continue;
1954
1955 err = _i915_vma_move_to_active(vma, eb->requests[j],
1956 j ? NULL :
1957 eb->composite_fence ?
1958 eb->composite_fence :
1959 &eb->requests[j]->fence,
1960 flags | __EXEC_OBJECT_NO_RESERVE);
1961 }
1962 }
1963
1964#ifdef CONFIG_MMU_NOTIFIER
1965 if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) {
1966 read_lock(&eb->i915->mm.notifier_lock);
1967
1968
1969
1970
1971
1972 for (i = 0; i < count; i++) {
1973 struct eb_vma *ev = &eb->vma[i];
1974 struct drm_i915_gem_object *obj = ev->vma->obj;
1975
1976 if (!i915_gem_object_is_userptr(obj))
1977 continue;
1978
1979 err = i915_gem_object_userptr_submit_done(obj);
1980 if (err)
1981 break;
1982 }
1983
1984 read_unlock(&eb->i915->mm.notifier_lock);
1985 }
1986#endif
1987
1988 if (unlikely(err))
1989 goto err_skip;
1990
1991
1992 intel_gt_chipset_flush(eb->gt);
1993 return 0;
1994
1995err_skip:
1996 for_each_batch_create_order(eb, j) {
1997 if (!eb->requests[j])
1998 break;
1999
2000 i915_request_set_error_once(eb->requests[j], err);
2001 }
2002 return err;
2003}
2004
2005static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
2006{
2007 if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
2008 return -EINVAL;
2009
2010
2011 if (!(exec->flags & (I915_EXEC_FENCE_ARRAY |
2012 I915_EXEC_USE_EXTENSIONS))) {
2013 if (exec->num_cliprects || exec->cliprects_ptr)
2014 return -EINVAL;
2015 }
2016
2017 if (exec->DR4 == 0xffffffff) {
2018 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
2019 exec->DR4 = 0;
2020 }
2021 if (exec->DR1 || exec->DR4)
2022 return -EINVAL;
2023
2024 if ((exec->batch_start_offset | exec->batch_len) & 0x7)
2025 return -EINVAL;
2026
2027 return 0;
2028}
2029
2030static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
2031{
2032 u32 *cs;
2033 int i;
2034
2035 if (GRAPHICS_VER(rq->engine->i915) != 7 || rq->engine->id != RCS0) {
2036 drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n");
2037 return -EINVAL;
2038 }
2039
2040 cs = intel_ring_begin(rq, 4 * 2 + 2);
2041 if (IS_ERR(cs))
2042 return PTR_ERR(cs);
2043
2044 *cs++ = MI_LOAD_REGISTER_IMM(4);
2045 for (i = 0; i < 4; i++) {
2046 *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
2047 *cs++ = 0;
2048 }
2049 *cs++ = MI_NOOP;
2050 intel_ring_advance(rq, cs);
2051
2052 return 0;
2053}
2054
2055static struct i915_vma *
2056shadow_batch_pin(struct i915_execbuffer *eb,
2057 struct drm_i915_gem_object *obj,
2058 struct i915_address_space *vm,
2059 unsigned int flags)
2060{
2061 struct i915_vma *vma;
2062 int err;
2063
2064 vma = i915_vma_instance(obj, vm, NULL);
2065 if (IS_ERR(vma))
2066 return vma;
2067
2068 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags);
2069 if (err)
2070 return ERR_PTR(err);
2071
2072 return vma;
2073}
2074
2075static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma)
2076{
2077
2078
2079
2080
2081 if (eb->batch_flags & I915_DISPATCH_SECURE)
2082 return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0);
2083
2084 return NULL;
2085}
2086
2087static int eb_parse(struct i915_execbuffer *eb)
2088{
2089 struct drm_i915_private *i915 = eb->i915;
2090 struct intel_gt_buffer_pool_node *pool = eb->batch_pool;
2091 struct i915_vma *shadow, *trampoline, *batch;
2092 unsigned long len;
2093 int err;
2094
2095 if (!eb_use_cmdparser(eb)) {
2096 batch = eb_dispatch_secure(eb, eb->batches[0]->vma);
2097 if (IS_ERR(batch))
2098 return PTR_ERR(batch);
2099
2100 goto secure_batch;
2101 }
2102
2103 if (intel_context_is_parallel(eb->context))
2104 return -EINVAL;
2105
2106 len = eb->batch_len[0];
2107 if (!CMDPARSER_USES_GGTT(eb->i915)) {
2108
2109
2110
2111
2112 if (!eb->context->vm->has_read_only) {
2113 drm_dbg(&i915->drm,
2114 "Cannot prevent post-scan tampering without RO capable vm\n");
2115 return -EINVAL;
2116 }
2117 } else {
2118 len += I915_CMD_PARSER_TRAMPOLINE_SIZE;
2119 }
2120 if (unlikely(len < eb->batch_len[0]))
2121 return -EINVAL;
2122
2123 if (!pool) {
2124 pool = intel_gt_get_buffer_pool(eb->gt, len,
2125 I915_MAP_WB);
2126 if (IS_ERR(pool))
2127 return PTR_ERR(pool);
2128 eb->batch_pool = pool;
2129 }
2130
2131 err = i915_gem_object_lock(pool->obj, &eb->ww);
2132 if (err)
2133 goto err;
2134
2135 shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER);
2136 if (IS_ERR(shadow)) {
2137 err = PTR_ERR(shadow);
2138 goto err;
2139 }
2140 intel_gt_buffer_pool_mark_used(pool);
2141 i915_gem_object_set_readonly(shadow->obj);
2142 shadow->private = pool;
2143
2144 trampoline = NULL;
2145 if (CMDPARSER_USES_GGTT(eb->i915)) {
2146 trampoline = shadow;
2147
2148 shadow = shadow_batch_pin(eb, pool->obj,
2149 &eb->gt->ggtt->vm,
2150 PIN_GLOBAL);
2151 if (IS_ERR(shadow)) {
2152 err = PTR_ERR(shadow);
2153 shadow = trampoline;
2154 goto err_shadow;
2155 }
2156 shadow->private = pool;
2157
2158 eb->batch_flags |= I915_DISPATCH_SECURE;
2159 }
2160
2161 batch = eb_dispatch_secure(eb, shadow);
2162 if (IS_ERR(batch)) {
2163 err = PTR_ERR(batch);
2164 goto err_trampoline;
2165 }
2166
2167 err = dma_resv_reserve_shared(shadow->resv, 1);
2168 if (err)
2169 goto err_trampoline;
2170
2171 err = intel_engine_cmd_parser(eb->context->engine,
2172 eb->batches[0]->vma,
2173 eb->batch_start_offset,
2174 eb->batch_len[0],
2175 shadow, trampoline);
2176 if (err)
2177 goto err_unpin_batch;
2178
2179 eb->batches[0] = &eb->vma[eb->buffer_count++];
2180 eb->batches[0]->vma = i915_vma_get(shadow);
2181 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN;
2182
2183 eb->trampoline = trampoline;
2184 eb->batch_start_offset = 0;
2185
2186secure_batch:
2187 if (batch) {
2188 if (intel_context_is_parallel(eb->context))
2189 return -EINVAL;
2190
2191 eb->batches[0] = &eb->vma[eb->buffer_count++];
2192 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN;
2193 eb->batches[0]->vma = i915_vma_get(batch);
2194 }
2195 return 0;
2196
2197err_unpin_batch:
2198 if (batch)
2199 i915_vma_unpin(batch);
2200err_trampoline:
2201 if (trampoline)
2202 i915_vma_unpin(trampoline);
2203err_shadow:
2204 i915_vma_unpin(shadow);
2205err:
2206 return err;
2207}
2208
2209static int eb_request_submit(struct i915_execbuffer *eb,
2210 struct i915_request *rq,
2211 struct i915_vma *batch,
2212 u64 batch_len)
2213{
2214 int err;
2215
2216 if (intel_context_nopreempt(rq->context))
2217 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
2218
2219 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
2220 err = i915_reset_gen7_sol_offsets(rq);
2221 if (err)
2222 return err;
2223 }
2224
2225
2226
2227
2228
2229
2230
2231 if (rq->context->engine->emit_init_breadcrumb) {
2232 err = rq->context->engine->emit_init_breadcrumb(rq);
2233 if (err)
2234 return err;
2235 }
2236
2237 err = rq->context->engine->emit_bb_start(rq,
2238 batch->node.start +
2239 eb->batch_start_offset,
2240 batch_len,
2241 eb->batch_flags);
2242 if (err)
2243 return err;
2244
2245 if (eb->trampoline) {
2246 GEM_BUG_ON(intel_context_is_parallel(rq->context));
2247 GEM_BUG_ON(eb->batch_start_offset);
2248 err = rq->context->engine->emit_bb_start(rq,
2249 eb->trampoline->node.start +
2250 batch_len, 0, 0);
2251 if (err)
2252 return err;
2253 }
2254
2255 return 0;
2256}
2257
2258static int eb_submit(struct i915_execbuffer *eb)
2259{
2260 unsigned int i;
2261 int err;
2262
2263 err = eb_move_to_gpu(eb);
2264
2265 for_each_batch_create_order(eb, i) {
2266 if (!eb->requests[i])
2267 break;
2268
2269 trace_i915_request_queue(eb->requests[i], eb->batch_flags);
2270 if (!err)
2271 err = eb_request_submit(eb, eb->requests[i],
2272 eb->batches[i]->vma,
2273 eb->batch_len[i]);
2274 }
2275
2276 return err;
2277}
2278
2279static int num_vcs_engines(const struct drm_i915_private *i915)
2280{
2281 return hweight_long(VDBOX_MASK(&i915->gt));
2282}
2283
2284
2285
2286
2287
2288static unsigned int
2289gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
2290 struct drm_file *file)
2291{
2292 struct drm_i915_file_private *file_priv = file->driver_priv;
2293
2294
2295 if ((int)file_priv->bsd_engine < 0)
2296 file_priv->bsd_engine =
2297 get_random_int() % num_vcs_engines(dev_priv);
2298
2299 return file_priv->bsd_engine;
2300}
2301
2302static const enum intel_engine_id user_ring_map[] = {
2303 [I915_EXEC_DEFAULT] = RCS0,
2304 [I915_EXEC_RENDER] = RCS0,
2305 [I915_EXEC_BLT] = BCS0,
2306 [I915_EXEC_BSD] = VCS0,
2307 [I915_EXEC_VEBOX] = VECS0
2308};
2309
2310static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce)
2311{
2312 struct intel_ring *ring = ce->ring;
2313 struct intel_timeline *tl = ce->timeline;
2314 struct i915_request *rq;
2315
2316
2317
2318
2319
2320 if (intel_ring_update_space(ring) >= PAGE_SIZE)
2321 return NULL;
2322
2323
2324
2325
2326
2327
2328
2329
2330 list_for_each_entry(rq, &tl->requests, link) {
2331 if (rq->ring != ring)
2332 continue;
2333
2334 if (__intel_ring_space(rq->postfix,
2335 ring->emit, ring->size) > ring->size / 2)
2336 break;
2337 }
2338 if (&rq->link == &tl->requests)
2339 return NULL;
2340
2341 return i915_request_get(rq);
2342}
2343
2344static int eb_pin_timeline(struct i915_execbuffer *eb, struct intel_context *ce,
2345 bool throttle)
2346{
2347 struct intel_timeline *tl;
2348 struct i915_request *rq = NULL;
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358 tl = intel_context_timeline_lock(ce);
2359 if (IS_ERR(tl))
2360 return PTR_ERR(tl);
2361
2362 intel_context_enter(ce);
2363 if (throttle)
2364 rq = eb_throttle(eb, ce);
2365 intel_context_timeline_unlock(tl);
2366
2367 if (rq) {
2368 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK;
2369 long timeout = nonblock ? 0 : MAX_SCHEDULE_TIMEOUT;
2370
2371 if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE,
2372 timeout) < 0) {
2373 i915_request_put(rq);
2374
2375 tl = intel_context_timeline_lock(ce);
2376 intel_context_exit(ce);
2377 intel_context_timeline_unlock(tl);
2378
2379 if (nonblock)
2380 return -EWOULDBLOCK;
2381 else
2382 return -EINTR;
2383 }
2384 i915_request_put(rq);
2385 }
2386
2387 return 0;
2388}
2389
2390static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle)
2391{
2392 struct intel_context *ce = eb->context, *child;
2393 int err;
2394 int i = 0, j = 0;
2395
2396 GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED);
2397
2398 if (unlikely(intel_context_is_banned(ce)))
2399 return -EIO;
2400
2401
2402
2403
2404
2405
2406 err = intel_context_pin_ww(ce, &eb->ww);
2407 if (err)
2408 return err;
2409 for_each_child(ce, child) {
2410 err = intel_context_pin_ww(child, &eb->ww);
2411 GEM_BUG_ON(err);
2412 }
2413
2414 for_each_child(ce, child) {
2415 err = eb_pin_timeline(eb, child, throttle);
2416 if (err)
2417 goto unwind;
2418 ++i;
2419 }
2420 err = eb_pin_timeline(eb, ce, throttle);
2421 if (err)
2422 goto unwind;
2423
2424 eb->args->flags |= __EXEC_ENGINE_PINNED;
2425 return 0;
2426
2427unwind:
2428 for_each_child(ce, child) {
2429 if (j++ < i) {
2430 mutex_lock(&child->timeline->mutex);
2431 intel_context_exit(child);
2432 mutex_unlock(&child->timeline->mutex);
2433 }
2434 }
2435 for_each_child(ce, child)
2436 intel_context_unpin(child);
2437 intel_context_unpin(ce);
2438 return err;
2439}
2440
2441static void eb_unpin_engine(struct i915_execbuffer *eb)
2442{
2443 struct intel_context *ce = eb->context, *child;
2444
2445 if (!(eb->args->flags & __EXEC_ENGINE_PINNED))
2446 return;
2447
2448 eb->args->flags &= ~__EXEC_ENGINE_PINNED;
2449
2450 for_each_child(ce, child) {
2451 mutex_lock(&child->timeline->mutex);
2452 intel_context_exit(child);
2453 mutex_unlock(&child->timeline->mutex);
2454
2455 intel_context_unpin(child);
2456 }
2457
2458 mutex_lock(&ce->timeline->mutex);
2459 intel_context_exit(ce);
2460 mutex_unlock(&ce->timeline->mutex);
2461
2462 intel_context_unpin(ce);
2463}
2464
2465static unsigned int
2466eb_select_legacy_ring(struct i915_execbuffer *eb)
2467{
2468 struct drm_i915_private *i915 = eb->i915;
2469 struct drm_i915_gem_execbuffer2 *args = eb->args;
2470 unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
2471
2472 if (user_ring_id != I915_EXEC_BSD &&
2473 (args->flags & I915_EXEC_BSD_MASK)) {
2474 drm_dbg(&i915->drm,
2475 "execbuf with non bsd ring but with invalid "
2476 "bsd dispatch flags: %d\n", (int)(args->flags));
2477 return -1;
2478 }
2479
2480 if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) {
2481 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
2482
2483 if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
2484 bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file);
2485 } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
2486 bsd_idx <= I915_EXEC_BSD_RING2) {
2487 bsd_idx >>= I915_EXEC_BSD_SHIFT;
2488 bsd_idx--;
2489 } else {
2490 drm_dbg(&i915->drm,
2491 "execbuf with unknown bsd ring: %u\n",
2492 bsd_idx);
2493 return -1;
2494 }
2495
2496 return _VCS(bsd_idx);
2497 }
2498
2499 if (user_ring_id >= ARRAY_SIZE(user_ring_map)) {
2500 drm_dbg(&i915->drm, "execbuf with unknown ring: %u\n",
2501 user_ring_id);
2502 return -1;
2503 }
2504
2505 return user_ring_map[user_ring_id];
2506}
2507
2508static int
2509eb_select_engine(struct i915_execbuffer *eb)
2510{
2511 struct intel_context *ce, *child;
2512 unsigned int idx;
2513 int err;
2514
2515 if (i915_gem_context_user_engines(eb->gem_context))
2516 idx = eb->args->flags & I915_EXEC_RING_MASK;
2517 else
2518 idx = eb_select_legacy_ring(eb);
2519
2520 ce = i915_gem_context_get_engine(eb->gem_context, idx);
2521 if (IS_ERR(ce))
2522 return PTR_ERR(ce);
2523
2524 if (intel_context_is_parallel(ce)) {
2525 if (eb->buffer_count < ce->parallel.number_children + 1) {
2526 intel_context_put(ce);
2527 return -EINVAL;
2528 }
2529 if (eb->batch_start_offset || eb->args->batch_len) {
2530 intel_context_put(ce);
2531 return -EINVAL;
2532 }
2533 }
2534 eb->num_batches = ce->parallel.number_children + 1;
2535
2536 for_each_child(ce, child)
2537 intel_context_get(child);
2538 intel_gt_pm_get(ce->engine->gt);
2539
2540 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
2541 err = intel_context_alloc_state(ce);
2542 if (err)
2543 goto err;
2544 }
2545 for_each_child(ce, child) {
2546 if (!test_bit(CONTEXT_ALLOC_BIT, &child->flags)) {
2547 err = intel_context_alloc_state(child);
2548 if (err)
2549 goto err;
2550 }
2551 }
2552
2553
2554
2555
2556
2557 err = intel_gt_terminally_wedged(ce->engine->gt);
2558 if (err)
2559 goto err;
2560
2561 eb->context = ce;
2562 eb->gt = ce->engine->gt;
2563
2564
2565
2566
2567
2568
2569 return err;
2570
2571err:
2572 intel_gt_pm_put(ce->engine->gt);
2573 for_each_child(ce, child)
2574 intel_context_put(child);
2575 intel_context_put(ce);
2576 return err;
2577}
2578
2579static void
2580eb_put_engine(struct i915_execbuffer *eb)
2581{
2582 struct intel_context *child;
2583
2584 intel_gt_pm_put(eb->gt);
2585 for_each_child(eb->context, child)
2586 intel_context_put(child);
2587 intel_context_put(eb->context);
2588}
2589
2590static void
2591__free_fence_array(struct eb_fence *fences, unsigned int n)
2592{
2593 while (n--) {
2594 drm_syncobj_put(ptr_mask_bits(fences[n].syncobj, 2));
2595 dma_fence_put(fences[n].dma_fence);
2596 dma_fence_chain_free(fences[n].chain_fence);
2597 }
2598 kvfree(fences);
2599}
2600
2601static int
2602add_timeline_fence_array(struct i915_execbuffer *eb,
2603 const struct drm_i915_gem_execbuffer_ext_timeline_fences *timeline_fences)
2604{
2605 struct drm_i915_gem_exec_fence __user *user_fences;
2606 u64 __user *user_values;
2607 struct eb_fence *f;
2608 u64 nfences;
2609 int err = 0;
2610
2611 nfences = timeline_fences->fence_count;
2612 if (!nfences)
2613 return 0;
2614
2615
2616 BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2617 if (nfences > min_t(unsigned long,
2618 ULONG_MAX / sizeof(*user_fences),
2619 SIZE_MAX / sizeof(*f)) - eb->num_fences)
2620 return -EINVAL;
2621
2622 user_fences = u64_to_user_ptr(timeline_fences->handles_ptr);
2623 if (!access_ok(user_fences, nfences * sizeof(*user_fences)))
2624 return -EFAULT;
2625
2626 user_values = u64_to_user_ptr(timeline_fences->values_ptr);
2627 if (!access_ok(user_values, nfences * sizeof(*user_values)))
2628 return -EFAULT;
2629
2630 f = krealloc(eb->fences,
2631 (eb->num_fences + nfences) * sizeof(*f),
2632 __GFP_NOWARN | GFP_KERNEL);
2633 if (!f)
2634 return -ENOMEM;
2635
2636 eb->fences = f;
2637 f += eb->num_fences;
2638
2639 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
2640 ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
2641
2642 while (nfences--) {
2643 struct drm_i915_gem_exec_fence user_fence;
2644 struct drm_syncobj *syncobj;
2645 struct dma_fence *fence = NULL;
2646 u64 point;
2647
2648 if (__copy_from_user(&user_fence,
2649 user_fences++,
2650 sizeof(user_fence)))
2651 return -EFAULT;
2652
2653 if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
2654 return -EINVAL;
2655
2656 if (__get_user(point, user_values++))
2657 return -EFAULT;
2658
2659 syncobj = drm_syncobj_find(eb->file, user_fence.handle);
2660 if (!syncobj) {
2661 DRM_DEBUG("Invalid syncobj handle provided\n");
2662 return -ENOENT;
2663 }
2664
2665 fence = drm_syncobj_fence_get(syncobj);
2666
2667 if (!fence && user_fence.flags &&
2668 !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2669 DRM_DEBUG("Syncobj handle has no fence\n");
2670 drm_syncobj_put(syncobj);
2671 return -EINVAL;
2672 }
2673
2674 if (fence)
2675 err = dma_fence_chain_find_seqno(&fence, point);
2676
2677 if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2678 DRM_DEBUG("Syncobj handle missing requested point %llu\n", point);
2679 dma_fence_put(fence);
2680 drm_syncobj_put(syncobj);
2681 return err;
2682 }
2683
2684
2685
2686
2687
2688
2689 if (!fence && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2690 drm_syncobj_put(syncobj);
2691 continue;
2692 }
2693
2694
2695
2696
2697
2698 if (point != 0 && user_fence.flags & I915_EXEC_FENCE_SIGNAL) {
2699
2700
2701
2702
2703 if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
2704 DRM_DEBUG("Trying to wait & signal the same timeline point.\n");
2705 dma_fence_put(fence);
2706 drm_syncobj_put(syncobj);
2707 return -EINVAL;
2708 }
2709
2710 f->chain_fence = dma_fence_chain_alloc();
2711 if (!f->chain_fence) {
2712 drm_syncobj_put(syncobj);
2713 dma_fence_put(fence);
2714 return -ENOMEM;
2715 }
2716 } else {
2717 f->chain_fence = NULL;
2718 }
2719
2720 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
2721 f->dma_fence = fence;
2722 f->value = point;
2723 f++;
2724 eb->num_fences++;
2725 }
2726
2727 return 0;
2728}
2729
2730static int add_fence_array(struct i915_execbuffer *eb)
2731{
2732 struct drm_i915_gem_execbuffer2 *args = eb->args;
2733 struct drm_i915_gem_exec_fence __user *user;
2734 unsigned long num_fences = args->num_cliprects;
2735 struct eb_fence *f;
2736
2737 if (!(args->flags & I915_EXEC_FENCE_ARRAY))
2738 return 0;
2739
2740 if (!num_fences)
2741 return 0;
2742
2743
2744 BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2745 if (num_fences > min_t(unsigned long,
2746 ULONG_MAX / sizeof(*user),
2747 SIZE_MAX / sizeof(*f) - eb->num_fences))
2748 return -EINVAL;
2749
2750 user = u64_to_user_ptr(args->cliprects_ptr);
2751 if (!access_ok(user, num_fences * sizeof(*user)))
2752 return -EFAULT;
2753
2754 f = krealloc(eb->fences,
2755 (eb->num_fences + num_fences) * sizeof(*f),
2756 __GFP_NOWARN | GFP_KERNEL);
2757 if (!f)
2758 return -ENOMEM;
2759
2760 eb->fences = f;
2761 f += eb->num_fences;
2762 while (num_fences--) {
2763 struct drm_i915_gem_exec_fence user_fence;
2764 struct drm_syncobj *syncobj;
2765 struct dma_fence *fence = NULL;
2766
2767 if (__copy_from_user(&user_fence, user++, sizeof(user_fence)))
2768 return -EFAULT;
2769
2770 if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
2771 return -EINVAL;
2772
2773 syncobj = drm_syncobj_find(eb->file, user_fence.handle);
2774 if (!syncobj) {
2775 DRM_DEBUG("Invalid syncobj handle provided\n");
2776 return -ENOENT;
2777 }
2778
2779 if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
2780 fence = drm_syncobj_fence_get(syncobj);
2781 if (!fence) {
2782 DRM_DEBUG("Syncobj handle has no fence\n");
2783 drm_syncobj_put(syncobj);
2784 return -EINVAL;
2785 }
2786 }
2787
2788 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
2789 ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
2790
2791 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
2792 f->dma_fence = fence;
2793 f->value = 0;
2794 f->chain_fence = NULL;
2795 f++;
2796 eb->num_fences++;
2797 }
2798
2799 return 0;
2800}
2801
2802static void put_fence_array(struct eb_fence *fences, int num_fences)
2803{
2804 if (fences)
2805 __free_fence_array(fences, num_fences);
2806}
2807
2808static int
2809await_fence_array(struct i915_execbuffer *eb,
2810 struct i915_request *rq)
2811{
2812 unsigned int n;
2813 int err;
2814
2815 for (n = 0; n < eb->num_fences; n++) {
2816 struct drm_syncobj *syncobj;
2817 unsigned int flags;
2818
2819 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2);
2820
2821 if (!eb->fences[n].dma_fence)
2822 continue;
2823
2824 err = i915_request_await_dma_fence(rq, eb->fences[n].dma_fence);
2825 if (err < 0)
2826 return err;
2827 }
2828
2829 return 0;
2830}
2831
2832static void signal_fence_array(const struct i915_execbuffer *eb,
2833 struct dma_fence * const fence)
2834{
2835 unsigned int n;
2836
2837 for (n = 0; n < eb->num_fences; n++) {
2838 struct drm_syncobj *syncobj;
2839 unsigned int flags;
2840
2841 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2);
2842 if (!(flags & I915_EXEC_FENCE_SIGNAL))
2843 continue;
2844
2845 if (eb->fences[n].chain_fence) {
2846 drm_syncobj_add_point(syncobj,
2847 eb->fences[n].chain_fence,
2848 fence,
2849 eb->fences[n].value);
2850
2851
2852
2853
2854 eb->fences[n].chain_fence = NULL;
2855 } else {
2856 drm_syncobj_replace_fence(syncobj, fence);
2857 }
2858 }
2859}
2860
2861static int
2862parse_timeline_fences(struct i915_user_extension __user *ext, void *data)
2863{
2864 struct i915_execbuffer *eb = data;
2865 struct drm_i915_gem_execbuffer_ext_timeline_fences timeline_fences;
2866
2867 if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences)))
2868 return -EFAULT;
2869
2870 return add_timeline_fence_array(eb, &timeline_fences);
2871}
2872
2873static void retire_requests(struct intel_timeline *tl, struct i915_request *end)
2874{
2875 struct i915_request *rq, *rn;
2876
2877 list_for_each_entry_safe(rq, rn, &tl->requests, link)
2878 if (rq == end || !i915_request_retire(rq))
2879 break;
2880}
2881
2882static int eb_request_add(struct i915_execbuffer *eb, struct i915_request *rq,
2883 int err, bool last_parallel)
2884{
2885 struct intel_timeline * const tl = i915_request_timeline(rq);
2886 struct i915_sched_attr attr = {};
2887 struct i915_request *prev;
2888
2889 lockdep_assert_held(&tl->mutex);
2890 lockdep_unpin_lock(&tl->mutex, rq->cookie);
2891
2892 trace_i915_request_add(rq);
2893
2894 prev = __i915_request_commit(rq);
2895
2896
2897 if (likely(!intel_context_is_closed(eb->context))) {
2898 attr = eb->gem_context->sched;
2899 } else {
2900
2901 i915_request_set_error_once(rq, -ENOENT);
2902 __i915_request_skip(rq);
2903 err = -ENOENT;
2904 }
2905
2906 if (intel_context_is_parallel(eb->context)) {
2907 if (err) {
2908 __i915_request_skip(rq);
2909 set_bit(I915_FENCE_FLAG_SKIP_PARALLEL,
2910 &rq->fence.flags);
2911 }
2912 if (last_parallel)
2913 set_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL,
2914 &rq->fence.flags);
2915 }
2916
2917 __i915_request_queue(rq, &attr);
2918
2919
2920 if (prev)
2921 retire_requests(tl, prev);
2922
2923 mutex_unlock(&tl->mutex);
2924
2925 return err;
2926}
2927
2928static int eb_requests_add(struct i915_execbuffer *eb, int err)
2929{
2930 int i;
2931
2932
2933
2934
2935
2936 for_each_batch_add_order(eb, i) {
2937 struct i915_request *rq = eb->requests[i];
2938
2939 if (!rq)
2940 continue;
2941 err |= eb_request_add(eb, rq, err, i == 0);
2942 }
2943
2944 return err;
2945}
2946
2947static const i915_user_extension_fn execbuf_extensions[] = {
2948 [DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES] = parse_timeline_fences,
2949};
2950
2951static int
2952parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args,
2953 struct i915_execbuffer *eb)
2954{
2955 if (!(args->flags & I915_EXEC_USE_EXTENSIONS))
2956 return 0;
2957
2958
2959
2960
2961 if (eb->args->flags & I915_EXEC_FENCE_ARRAY)
2962 return -EINVAL;
2963
2964 if (args->num_cliprects != 0)
2965 return -EINVAL;
2966
2967 return i915_user_extensions(u64_to_user_ptr(args->cliprects_ptr),
2968 execbuf_extensions,
2969 ARRAY_SIZE(execbuf_extensions),
2970 eb);
2971}
2972
2973static void eb_requests_get(struct i915_execbuffer *eb)
2974{
2975 unsigned int i;
2976
2977 for_each_batch_create_order(eb, i) {
2978 if (!eb->requests[i])
2979 break;
2980
2981 i915_request_get(eb->requests[i]);
2982 }
2983}
2984
2985static void eb_requests_put(struct i915_execbuffer *eb)
2986{
2987 unsigned int i;
2988
2989 for_each_batch_create_order(eb, i) {
2990 if (!eb->requests[i])
2991 break;
2992
2993 i915_request_put(eb->requests[i]);
2994 }
2995}
2996
2997static struct sync_file *
2998eb_composite_fence_create(struct i915_execbuffer *eb, int out_fence_fd)
2999{
3000 struct sync_file *out_fence = NULL;
3001 struct dma_fence_array *fence_array;
3002 struct dma_fence **fences;
3003 unsigned int i;
3004
3005 GEM_BUG_ON(!intel_context_is_parent(eb->context));
3006
3007 fences = kmalloc_array(eb->num_batches, sizeof(*fences), GFP_KERNEL);
3008 if (!fences)
3009 return ERR_PTR(-ENOMEM);
3010
3011 for_each_batch_create_order(eb, i) {
3012 fences[i] = &eb->requests[i]->fence;
3013 __set_bit(I915_FENCE_FLAG_COMPOSITE,
3014 &eb->requests[i]->fence.flags);
3015 }
3016
3017 fence_array = dma_fence_array_create(eb->num_batches,
3018 fences,
3019 eb->context->parallel.fence_context,
3020 eb->context->parallel.seqno++,
3021 false);
3022 if (!fence_array) {
3023 kfree(fences);
3024 return ERR_PTR(-ENOMEM);
3025 }
3026
3027
3028 for_each_batch_create_order(eb, i)
3029 dma_fence_get(fences[i]);
3030
3031 if (out_fence_fd != -1) {
3032 out_fence = sync_file_create(&fence_array->base);
3033
3034 dma_fence_put(&fence_array->base);
3035 if (!out_fence)
3036 return ERR_PTR(-ENOMEM);
3037 }
3038
3039 eb->composite_fence = &fence_array->base;
3040
3041 return out_fence;
3042}
3043
3044static struct sync_file *
3045eb_fences_add(struct i915_execbuffer *eb, struct i915_request *rq,
3046 struct dma_fence *in_fence, int out_fence_fd)
3047{
3048 struct sync_file *out_fence = NULL;
3049 int err;
3050
3051 if (unlikely(eb->gem_context->syncobj)) {
3052 struct dma_fence *fence;
3053
3054 fence = drm_syncobj_fence_get(eb->gem_context->syncobj);
3055 err = i915_request_await_dma_fence(rq, fence);
3056 dma_fence_put(fence);
3057 if (err)
3058 return ERR_PTR(err);
3059 }
3060
3061 if (in_fence) {
3062 if (eb->args->flags & I915_EXEC_FENCE_SUBMIT)
3063 err = i915_request_await_execution(rq, in_fence);
3064 else
3065 err = i915_request_await_dma_fence(rq, in_fence);
3066 if (err < 0)
3067 return ERR_PTR(err);
3068 }
3069
3070 if (eb->fences) {
3071 err = await_fence_array(eb, rq);
3072 if (err)
3073 return ERR_PTR(err);
3074 }
3075
3076 if (intel_context_is_parallel(eb->context)) {
3077 out_fence = eb_composite_fence_create(eb, out_fence_fd);
3078 if (IS_ERR(out_fence))
3079 return ERR_PTR(-ENOMEM);
3080 } else if (out_fence_fd != -1) {
3081 out_fence = sync_file_create(&rq->fence);
3082 if (!out_fence)
3083 return ERR_PTR(-ENOMEM);
3084 }
3085
3086 return out_fence;
3087}
3088
3089static struct intel_context *
3090eb_find_context(struct i915_execbuffer *eb, unsigned int context_number)
3091{
3092 struct intel_context *child;
3093
3094 if (likely(context_number == 0))
3095 return eb->context;
3096
3097 for_each_child(eb->context, child)
3098 if (!--context_number)
3099 return child;
3100
3101 GEM_BUG_ON("Context not found");
3102
3103 return NULL;
3104}
3105
3106static struct sync_file *
3107eb_requests_create(struct i915_execbuffer *eb, struct dma_fence *in_fence,
3108 int out_fence_fd)
3109{
3110 struct sync_file *out_fence = NULL;
3111 unsigned int i;
3112
3113 for_each_batch_create_order(eb, i) {
3114
3115 eb->requests[i] = i915_request_create(eb_find_context(eb, i));
3116 if (IS_ERR(eb->requests[i])) {
3117 out_fence = ERR_PTR(PTR_ERR(eb->requests[i]));
3118 eb->requests[i] = NULL;
3119 return out_fence;
3120 }
3121
3122
3123
3124
3125
3126
3127 if (i + 1 == eb->num_batches) {
3128 out_fence = eb_fences_add(eb, eb->requests[i],
3129 in_fence, out_fence_fd);
3130 if (IS_ERR(out_fence))
3131 return out_fence;
3132 }
3133
3134
3135
3136
3137
3138
3139
3140
3141 eb->requests[i]->batch = eb->batches[i]->vma;
3142 if (eb->batch_pool) {
3143 GEM_BUG_ON(intel_context_is_parallel(eb->context));
3144 intel_gt_buffer_pool_mark_active(eb->batch_pool,
3145 eb->requests[i]);
3146 }
3147 }
3148
3149 return out_fence;
3150}
3151
3152static int
3153i915_gem_do_execbuffer(struct drm_device *dev,
3154 struct drm_file *file,
3155 struct drm_i915_gem_execbuffer2 *args,
3156 struct drm_i915_gem_exec_object2 *exec)
3157{
3158 struct drm_i915_private *i915 = to_i915(dev);
3159 struct i915_execbuffer eb;
3160 struct dma_fence *in_fence = NULL;
3161 struct sync_file *out_fence = NULL;
3162 int out_fence_fd = -1;
3163 int err;
3164
3165 BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS);
3166 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
3167 ~__EXEC_OBJECT_UNKNOWN_FLAGS);
3168
3169 eb.i915 = i915;
3170 eb.file = file;
3171 eb.args = args;
3172 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
3173 args->flags |= __EXEC_HAS_RELOC;
3174
3175 eb.exec = exec;
3176 eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1);
3177 eb.vma[0].vma = NULL;
3178 eb.batch_pool = NULL;
3179
3180 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
3181 reloc_cache_init(&eb.reloc_cache, eb.i915);
3182
3183 eb.buffer_count = args->buffer_count;
3184 eb.batch_start_offset = args->batch_start_offset;
3185 eb.trampoline = NULL;
3186
3187 eb.fences = NULL;
3188 eb.num_fences = 0;
3189
3190 memset(eb.requests, 0, sizeof(struct i915_request *) *
3191 ARRAY_SIZE(eb.requests));
3192 eb.composite_fence = NULL;
3193
3194 eb.batch_flags = 0;
3195 if (args->flags & I915_EXEC_SECURE) {
3196 if (GRAPHICS_VER(i915) >= 11)
3197 return -ENODEV;
3198
3199
3200 if (!HAS_SECURE_BATCHES(i915))
3201 return -EPERM;
3202
3203 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
3204 return -EPERM;
3205
3206 eb.batch_flags |= I915_DISPATCH_SECURE;
3207 }
3208 if (args->flags & I915_EXEC_IS_PINNED)
3209 eb.batch_flags |= I915_DISPATCH_PINNED;
3210
3211 err = parse_execbuf2_extensions(args, &eb);
3212 if (err)
3213 goto err_ext;
3214
3215 err = add_fence_array(&eb);
3216 if (err)
3217 goto err_ext;
3218
3219#define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT)
3220 if (args->flags & IN_FENCES) {
3221 if ((args->flags & IN_FENCES) == IN_FENCES)
3222 return -EINVAL;
3223
3224 in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
3225 if (!in_fence) {
3226 err = -EINVAL;
3227 goto err_ext;
3228 }
3229 }
3230#undef IN_FENCES
3231
3232 if (args->flags & I915_EXEC_FENCE_OUT) {
3233 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
3234 if (out_fence_fd < 0) {
3235 err = out_fence_fd;
3236 goto err_in_fence;
3237 }
3238 }
3239
3240 err = eb_create(&eb);
3241 if (err)
3242 goto err_out_fence;
3243
3244 GEM_BUG_ON(!eb.lut_size);
3245
3246 err = eb_select_context(&eb);
3247 if (unlikely(err))
3248 goto err_destroy;
3249
3250 err = eb_select_engine(&eb);
3251 if (unlikely(err))
3252 goto err_context;
3253
3254 err = eb_lookup_vmas(&eb);
3255 if (err) {
3256 eb_release_vmas(&eb, true);
3257 goto err_engine;
3258 }
3259
3260 i915_gem_ww_ctx_init(&eb.ww, true);
3261
3262 err = eb_relocate_parse(&eb);
3263 if (err) {
3264
3265
3266
3267
3268
3269
3270
3271 args->flags &= ~__EXEC_HAS_RELOC;
3272 goto err_vma;
3273 }
3274
3275 ww_acquire_done(&eb.ww.ctx);
3276
3277 out_fence = eb_requests_create(&eb, in_fence, out_fence_fd);
3278 if (IS_ERR(out_fence)) {
3279 err = PTR_ERR(out_fence);
3280 out_fence = NULL;
3281 if (eb.requests[0])
3282 goto err_request;
3283 else
3284 goto err_vma;
3285 }
3286
3287 err = eb_submit(&eb);
3288
3289err_request:
3290 eb_requests_get(&eb);
3291 err = eb_requests_add(&eb, err);
3292
3293 if (eb.fences)
3294 signal_fence_array(&eb, eb.composite_fence ?
3295 eb.composite_fence :
3296 &eb.requests[0]->fence);
3297
3298 if (out_fence) {
3299 if (err == 0) {
3300 fd_install(out_fence_fd, out_fence->file);
3301 args->rsvd2 &= GENMASK_ULL(31, 0);
3302 args->rsvd2 |= (u64)out_fence_fd << 32;
3303 out_fence_fd = -1;
3304 } else {
3305 fput(out_fence->file);
3306 }
3307 }
3308
3309 if (unlikely(eb.gem_context->syncobj)) {
3310 drm_syncobj_replace_fence(eb.gem_context->syncobj,
3311 eb.composite_fence ?
3312 eb.composite_fence :
3313 &eb.requests[0]->fence);
3314 }
3315
3316 if (!out_fence && eb.composite_fence)
3317 dma_fence_put(eb.composite_fence);
3318
3319 eb_requests_put(&eb);
3320
3321err_vma:
3322 eb_release_vmas(&eb, true);
3323 if (eb.trampoline)
3324 i915_vma_unpin(eb.trampoline);
3325 WARN_ON(err == -EDEADLK);
3326 i915_gem_ww_ctx_fini(&eb.ww);
3327
3328 if (eb.batch_pool)
3329 intel_gt_buffer_pool_put(eb.batch_pool);
3330err_engine:
3331 eb_put_engine(&eb);
3332err_context:
3333 i915_gem_context_put(eb.gem_context);
3334err_destroy:
3335 eb_destroy(&eb);
3336err_out_fence:
3337 if (out_fence_fd != -1)
3338 put_unused_fd(out_fence_fd);
3339err_in_fence:
3340 dma_fence_put(in_fence);
3341err_ext:
3342 put_fence_array(eb.fences, eb.num_fences);
3343 return err;
3344}
3345
3346static size_t eb_element_size(void)
3347{
3348 return sizeof(struct drm_i915_gem_exec_object2) + sizeof(struct eb_vma);
3349}
3350
3351static bool check_buffer_count(size_t count)
3352{
3353 const size_t sz = eb_element_size();
3354
3355
3356
3357
3358
3359
3360
3361 return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
3362}
3363
3364int
3365i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
3366 struct drm_file *file)
3367{
3368 struct drm_i915_private *i915 = to_i915(dev);
3369 struct drm_i915_gem_execbuffer2 *args = data;
3370 struct drm_i915_gem_exec_object2 *exec2_list;
3371 const size_t count = args->buffer_count;
3372 int err;
3373
3374 if (!check_buffer_count(count)) {
3375 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count);
3376 return -EINVAL;
3377 }
3378
3379 err = i915_gem_check_execbuffer(args);
3380 if (err)
3381 return err;
3382
3383
3384 exec2_list = kvmalloc_array(count + 2, eb_element_size(),
3385 __GFP_NOWARN | GFP_KERNEL);
3386 if (exec2_list == NULL) {
3387 drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n",
3388 count);
3389 return -ENOMEM;
3390 }
3391 if (copy_from_user(exec2_list,
3392 u64_to_user_ptr(args->buffers_ptr),
3393 sizeof(*exec2_list) * count)) {
3394 drm_dbg(&i915->drm, "copy %zd exec entries failed\n", count);
3395 kvfree(exec2_list);
3396 return -EFAULT;
3397 }
3398
3399 err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
3400
3401
3402
3403
3404
3405
3406
3407 if (args->flags & __EXEC_HAS_RELOC) {
3408 struct drm_i915_gem_exec_object2 __user *user_exec_list =
3409 u64_to_user_ptr(args->buffers_ptr);
3410 unsigned int i;
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420 if (!user_write_access_begin(user_exec_list,
3421 count * sizeof(*user_exec_list)))
3422 goto end;
3423
3424 for (i = 0; i < args->buffer_count; i++) {
3425 if (!(exec2_list[i].offset & UPDATE))
3426 continue;
3427
3428 exec2_list[i].offset =
3429 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
3430 unsafe_put_user(exec2_list[i].offset,
3431 &user_exec_list[i].offset,
3432 end_user);
3433 }
3434end_user:
3435 user_write_access_end();
3436end:;
3437 }
3438
3439 args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
3440 kvfree(exec2_list);
3441 return err;
3442}
3443