1
2
3
4
5
6
7#include <linux/intel-iommu.h>
8#include <linux/dma-resv.h>
9#include <linux/sync_file.h>
10#include <linux/uaccess.h>
11
12#include <drm/drm_syncobj.h>
13
14#include "display/intel_frontbuffer.h"
15
16#include "gem/i915_gem_ioctls.h"
17#include "gt/intel_context.h"
18#include "gt/intel_gt.h"
19#include "gt/intel_gt_buffer_pool.h"
20#include "gt/intel_gt_pm.h"
21#include "gt/intel_ring.h"
22
23#include "i915_drv.h"
24#include "i915_gem_clflush.h"
25#include "i915_gem_context.h"
26#include "i915_gem_ioctls.h"
27#include "i915_sw_fence_work.h"
28#include "i915_trace.h"
29#include "i915_user_extensions.h"
30
31struct eb_vma {
32 struct i915_vma *vma;
33 unsigned int flags;
34
35
36 struct drm_i915_gem_exec_object2 *exec;
37 struct list_head bind_link;
38 struct list_head reloc_link;
39
40 struct hlist_node node;
41 u32 handle;
42};
43
44enum {
45 FORCE_CPU_RELOC = 1,
46 FORCE_GTT_RELOC,
47 FORCE_GPU_RELOC,
48#define DBG_FORCE_RELOC 0
49};
50
51#define __EXEC_OBJECT_HAS_PIN BIT(31)
52#define __EXEC_OBJECT_HAS_FENCE BIT(30)
53#define __EXEC_OBJECT_NEEDS_MAP BIT(29)
54#define __EXEC_OBJECT_NEEDS_BIAS BIT(28)
55#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 28)
56#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
57
58#define __EXEC_HAS_RELOC BIT(31)
59#define __EXEC_ENGINE_PINNED BIT(30)
60#define __EXEC_INTERNAL_FLAGS (~0u << 30)
61#define UPDATE PIN_OFFSET_FIXED
62
63#define BATCH_OFFSET_BIAS (256*1024)
64
65#define __I915_EXEC_ILLEGAL_FLAGS \
66 (__I915_EXEC_UNKNOWN_FLAGS | \
67 I915_EXEC_CONSTANTS_MASK | \
68 I915_EXEC_RESOURCE_STREAMER)
69
70
71#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
72#undef EINVAL
73#define EINVAL ({ \
74 DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \
75 22; \
76})
77#endif
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230struct eb_fence {
231 struct drm_syncobj *syncobj;
232 struct dma_fence *dma_fence;
233 u64 value;
234 struct dma_fence_chain *chain_fence;
235};
236
237struct i915_execbuffer {
238 struct drm_i915_private *i915;
239 struct drm_file *file;
240 struct drm_i915_gem_execbuffer2 *args;
241 struct drm_i915_gem_exec_object2 *exec;
242 struct eb_vma *vma;
243
244 struct intel_engine_cs *engine;
245 struct intel_context *context;
246 struct i915_gem_context *gem_context;
247
248 struct i915_request *request;
249 struct eb_vma *batch;
250 struct i915_vma *trampoline;
251
252
253 unsigned int buffer_count;
254
255
256 struct list_head unbound;
257
258
259 struct list_head relocs;
260
261 struct i915_gem_ww_ctx ww;
262
263
264
265
266
267
268 struct reloc_cache {
269 struct drm_mm_node node;
270 unsigned long vaddr;
271 unsigned long page;
272 unsigned int gen;
273 bool use_64bit_reloc : 1;
274 bool has_llc : 1;
275 bool has_fence : 1;
276 bool needs_unfenced : 1;
277
278 struct i915_request *rq;
279 u32 *rq_cmd;
280 unsigned int rq_size;
281 struct intel_gt_buffer_pool_node *pool;
282 } reloc_cache;
283
284 struct intel_gt_buffer_pool_node *reloc_pool;
285 struct intel_context *reloc_context;
286
287 u64 invalid_flags;
288 u32 context_flags;
289
290 u64 batch_len;
291 u32 batch_start_offset;
292 u32 batch_flags;
293 struct intel_gt_buffer_pool_node *batch_pool;
294
295
296
297
298
299
300 int lut_size;
301 struct hlist_head *buckets;
302
303 struct eb_fence *fences;
304 unsigned long num_fences;
305};
306
307static int eb_parse(struct i915_execbuffer *eb);
308static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb,
309 bool throttle);
310static void eb_unpin_engine(struct i915_execbuffer *eb);
311
312static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
313{
314 return intel_engine_requires_cmd_parser(eb->engine) ||
315 (intel_engine_using_cmd_parser(eb->engine) &&
316 eb->args->batch_len);
317}
318
319static int eb_create(struct i915_execbuffer *eb)
320{
321 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
322 unsigned int size = 1 + ilog2(eb->buffer_count);
323
324
325
326
327
328
329
330
331
332
333
334
335 do {
336 gfp_t flags;
337
338
339
340
341
342
343
344 flags = GFP_KERNEL;
345 if (size > 1)
346 flags |= __GFP_NORETRY | __GFP_NOWARN;
347
348 eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
349 flags);
350 if (eb->buckets)
351 break;
352 } while (--size);
353
354 if (unlikely(!size))
355 return -ENOMEM;
356
357 eb->lut_size = size;
358 } else {
359 eb->lut_size = -eb->buffer_count;
360 }
361
362 return 0;
363}
364
365static bool
366eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
367 const struct i915_vma *vma,
368 unsigned int flags)
369{
370 if (vma->node.size < entry->pad_to_size)
371 return true;
372
373 if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
374 return true;
375
376 if (flags & EXEC_OBJECT_PINNED &&
377 vma->node.start != entry->offset)
378 return true;
379
380 if (flags & __EXEC_OBJECT_NEEDS_BIAS &&
381 vma->node.start < BATCH_OFFSET_BIAS)
382 return true;
383
384 if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
385 (vma->node.start + vma->node.size - 1) >> 32)
386 return true;
387
388 if (flags & __EXEC_OBJECT_NEEDS_MAP &&
389 !i915_vma_is_map_and_fenceable(vma))
390 return true;
391
392 return false;
393}
394
395static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry,
396 unsigned int exec_flags)
397{
398 u64 pin_flags = 0;
399
400 if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
401 pin_flags |= PIN_GLOBAL;
402
403
404
405
406
407 if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
408 pin_flags |= PIN_ZONE_4G;
409
410 if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
411 pin_flags |= PIN_MAPPABLE;
412
413 if (exec_flags & EXEC_OBJECT_PINNED)
414 pin_flags |= entry->offset | PIN_OFFSET_FIXED;
415 else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS)
416 pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
417
418 return pin_flags;
419}
420
421static inline bool
422eb_pin_vma(struct i915_execbuffer *eb,
423 const struct drm_i915_gem_exec_object2 *entry,
424 struct eb_vma *ev)
425{
426 struct i915_vma *vma = ev->vma;
427 u64 pin_flags;
428
429 if (vma->node.size)
430 pin_flags = vma->node.start;
431 else
432 pin_flags = entry->offset & PIN_OFFSET_MASK;
433
434 pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
435 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT))
436 pin_flags |= PIN_GLOBAL;
437
438
439
440 if (unlikely(i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags))) {
441 if (entry->flags & EXEC_OBJECT_PINNED)
442 return false;
443
444
445 if (unlikely(i915_vma_pin_ww(vma, &eb->ww,
446 entry->pad_to_size,
447 entry->alignment,
448 eb_pin_flags(entry, ev->flags) |
449 PIN_USER | PIN_NOEVICT)))
450 return false;
451 }
452
453 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
454 if (unlikely(i915_vma_pin_fence(vma))) {
455 i915_vma_unpin(vma);
456 return false;
457 }
458
459 if (vma->fence)
460 ev->flags |= __EXEC_OBJECT_HAS_FENCE;
461 }
462
463 ev->flags |= __EXEC_OBJECT_HAS_PIN;
464 return !eb_vma_misplaced(entry, vma, ev->flags);
465}
466
467static inline void
468eb_unreserve_vma(struct eb_vma *ev)
469{
470 if (!(ev->flags & __EXEC_OBJECT_HAS_PIN))
471 return;
472
473 if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE))
474 __i915_vma_unpin_fence(ev->vma);
475
476 __i915_vma_unpin(ev->vma);
477 ev->flags &= ~__EXEC_OBJECT_RESERVED;
478}
479
480static int
481eb_validate_vma(struct i915_execbuffer *eb,
482 struct drm_i915_gem_exec_object2 *entry,
483 struct i915_vma *vma)
484{
485 if (unlikely(entry->flags & eb->invalid_flags))
486 return -EINVAL;
487
488 if (unlikely(entry->alignment &&
489 !is_power_of_2_u64(entry->alignment)))
490 return -EINVAL;
491
492
493
494
495
496 if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
497 entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK)))
498 return -EINVAL;
499
500
501 if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
502 if (unlikely(offset_in_page(entry->pad_to_size)))
503 return -EINVAL;
504 } else {
505 entry->pad_to_size = 0;
506 }
507
508
509
510
511
512 entry->offset = gen8_noncanonical_addr(entry->offset);
513
514 if (!eb->reloc_cache.has_fence) {
515 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
516 } else {
517 if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
518 eb->reloc_cache.needs_unfenced) &&
519 i915_gem_object_is_tiled(vma->obj))
520 entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
521 }
522
523 if (!(entry->flags & EXEC_OBJECT_PINNED))
524 entry->flags |= eb->context_flags;
525
526 return 0;
527}
528
529static void
530eb_add_vma(struct i915_execbuffer *eb,
531 unsigned int i, unsigned batch_idx,
532 struct i915_vma *vma)
533{
534 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
535 struct eb_vma *ev = &eb->vma[i];
536
537 GEM_BUG_ON(i915_vma_is_closed(vma));
538
539 ev->vma = vma;
540 ev->exec = entry;
541 ev->flags = entry->flags;
542
543 if (eb->lut_size > 0) {
544 ev->handle = entry->handle;
545 hlist_add_head(&ev->node,
546 &eb->buckets[hash_32(entry->handle,
547 eb->lut_size)]);
548 }
549
550 if (entry->relocation_count)
551 list_add_tail(&ev->reloc_link, &eb->relocs);
552
553
554
555
556
557
558
559
560
561
562 if (i == batch_idx) {
563 if (entry->relocation_count &&
564 !(ev->flags & EXEC_OBJECT_PINNED))
565 ev->flags |= __EXEC_OBJECT_NEEDS_BIAS;
566 if (eb->reloc_cache.has_fence)
567 ev->flags |= EXEC_OBJECT_NEEDS_FENCE;
568
569 eb->batch = ev;
570 }
571}
572
573static inline int use_cpu_reloc(const struct reloc_cache *cache,
574 const struct drm_i915_gem_object *obj)
575{
576 if (!i915_gem_object_has_struct_page(obj))
577 return false;
578
579 if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
580 return true;
581
582 if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
583 return false;
584
585 return (cache->has_llc ||
586 obj->cache_dirty ||
587 obj->cache_level != I915_CACHE_NONE);
588}
589
590static int eb_reserve_vma(struct i915_execbuffer *eb,
591 struct eb_vma *ev,
592 u64 pin_flags)
593{
594 struct drm_i915_gem_exec_object2 *entry = ev->exec;
595 struct i915_vma *vma = ev->vma;
596 int err;
597
598 if (drm_mm_node_allocated(&vma->node) &&
599 eb_vma_misplaced(entry, vma, ev->flags)) {
600 err = i915_vma_unbind(vma);
601 if (err)
602 return err;
603 }
604
605 err = i915_vma_pin_ww(vma, &eb->ww,
606 entry->pad_to_size, entry->alignment,
607 eb_pin_flags(entry, ev->flags) | pin_flags);
608 if (err)
609 return err;
610
611 if (entry->offset != vma->node.start) {
612 entry->offset = vma->node.start | UPDATE;
613 eb->args->flags |= __EXEC_HAS_RELOC;
614 }
615
616 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
617 err = i915_vma_pin_fence(vma);
618 if (unlikely(err)) {
619 i915_vma_unpin(vma);
620 return err;
621 }
622
623 if (vma->fence)
624 ev->flags |= __EXEC_OBJECT_HAS_FENCE;
625 }
626
627 ev->flags |= __EXEC_OBJECT_HAS_PIN;
628 GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags));
629
630 return 0;
631}
632
633static int eb_reserve(struct i915_execbuffer *eb)
634{
635 const unsigned int count = eb->buffer_count;
636 unsigned int pin_flags = PIN_USER | PIN_NONBLOCK;
637 struct list_head last;
638 struct eb_vma *ev;
639 unsigned int i, pass;
640 int err = 0;
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655 pass = 0;
656 do {
657 list_for_each_entry(ev, &eb->unbound, bind_link) {
658 err = eb_reserve_vma(eb, ev, pin_flags);
659 if (err)
660 break;
661 }
662 if (err != -ENOSPC)
663 return err;
664
665
666 INIT_LIST_HEAD(&eb->unbound);
667 INIT_LIST_HEAD(&last);
668 for (i = 0; i < count; i++) {
669 unsigned int flags;
670
671 ev = &eb->vma[i];
672 flags = ev->flags;
673 if (flags & EXEC_OBJECT_PINNED &&
674 flags & __EXEC_OBJECT_HAS_PIN)
675 continue;
676
677 eb_unreserve_vma(ev);
678
679 if (flags & EXEC_OBJECT_PINNED)
680
681 list_add(&ev->bind_link, &eb->unbound);
682 else if (flags & __EXEC_OBJECT_NEEDS_MAP)
683
684 list_add_tail(&ev->bind_link, &eb->unbound);
685 else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
686
687 list_add(&ev->bind_link, &last);
688 else
689 list_add_tail(&ev->bind_link, &last);
690 }
691 list_splice_tail(&last, &eb->unbound);
692
693 switch (pass++) {
694 case 0:
695 break;
696
697 case 1:
698
699 mutex_lock(&eb->context->vm->mutex);
700 err = i915_gem_evict_vm(eb->context->vm);
701 mutex_unlock(&eb->context->vm->mutex);
702 if (err)
703 return err;
704 break;
705
706 default:
707 return -ENOSPC;
708 }
709
710 pin_flags = PIN_USER;
711 } while (1);
712}
713
714static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
715{
716 if (eb->args->flags & I915_EXEC_BATCH_FIRST)
717 return 0;
718 else
719 return eb->buffer_count - 1;
720}
721
722static int eb_select_context(struct i915_execbuffer *eb)
723{
724 struct i915_gem_context *ctx;
725
726 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
727 if (unlikely(!ctx))
728 return -ENOENT;
729
730 eb->gem_context = ctx;
731 if (rcu_access_pointer(ctx->vm))
732 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
733
734 eb->context_flags = 0;
735 if (test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags))
736 eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS;
737
738 return 0;
739}
740
741static int __eb_add_lut(struct i915_execbuffer *eb,
742 u32 handle, struct i915_vma *vma)
743{
744 struct i915_gem_context *ctx = eb->gem_context;
745 struct i915_lut_handle *lut;
746 int err;
747
748 lut = i915_lut_handle_alloc();
749 if (unlikely(!lut))
750 return -ENOMEM;
751
752 i915_vma_get(vma);
753 if (!atomic_fetch_inc(&vma->open_count))
754 i915_vma_reopen(vma);
755 lut->handle = handle;
756 lut->ctx = ctx;
757
758
759 err = -EINTR;
760 if (!mutex_lock_interruptible(&ctx->lut_mutex)) {
761 struct i915_address_space *vm = rcu_access_pointer(ctx->vm);
762
763 if (unlikely(vm && vma->vm != vm))
764 err = -EAGAIN;
765 else if (likely(!i915_gem_context_is_closed(ctx)))
766 err = radix_tree_insert(&ctx->handles_vma, handle, vma);
767 else
768 err = -ENOENT;
769 if (err == 0) {
770 struct drm_i915_gem_object *obj = vma->obj;
771
772 spin_lock(&obj->lut_lock);
773 if (idr_find(&eb->file->object_idr, handle) == obj) {
774 list_add(&lut->obj_link, &obj->lut_list);
775 } else {
776 radix_tree_delete(&ctx->handles_vma, handle);
777 err = -ENOENT;
778 }
779 spin_unlock(&obj->lut_lock);
780 }
781 mutex_unlock(&ctx->lut_mutex);
782 }
783 if (unlikely(err))
784 goto err;
785
786 return 0;
787
788err:
789 i915_vma_close(vma);
790 i915_vma_put(vma);
791 i915_lut_handle_free(lut);
792 return err;
793}
794
795static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle)
796{
797 struct i915_address_space *vm = eb->context->vm;
798
799 do {
800 struct drm_i915_gem_object *obj;
801 struct i915_vma *vma;
802 int err;
803
804 rcu_read_lock();
805 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle);
806 if (likely(vma && vma->vm == vm))
807 vma = i915_vma_tryget(vma);
808 rcu_read_unlock();
809 if (likely(vma))
810 return vma;
811
812 obj = i915_gem_object_lookup(eb->file, handle);
813 if (unlikely(!obj))
814 return ERR_PTR(-ENOENT);
815
816 vma = i915_vma_instance(obj, vm, NULL);
817 if (IS_ERR(vma)) {
818 i915_gem_object_put(obj);
819 return vma;
820 }
821
822 err = __eb_add_lut(eb, handle, vma);
823 if (likely(!err))
824 return vma;
825
826 i915_gem_object_put(obj);
827 if (err != -EEXIST)
828 return ERR_PTR(err);
829 } while (1);
830}
831
832static int eb_lookup_vmas(struct i915_execbuffer *eb)
833{
834 struct drm_i915_private *i915 = eb->i915;
835 unsigned int batch = eb_batch_index(eb);
836 unsigned int i;
837 int err = 0;
838
839 INIT_LIST_HEAD(&eb->relocs);
840
841 for (i = 0; i < eb->buffer_count; i++) {
842 struct i915_vma *vma;
843
844 vma = eb_lookup_vma(eb, eb->exec[i].handle);
845 if (IS_ERR(vma)) {
846 err = PTR_ERR(vma);
847 goto err;
848 }
849
850 err = eb_validate_vma(eb, &eb->exec[i], vma);
851 if (unlikely(err)) {
852 i915_vma_put(vma);
853 goto err;
854 }
855
856 eb_add_vma(eb, i, batch, vma);
857 }
858
859 if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)) {
860 drm_dbg(&i915->drm,
861 "Attempting to use self-modifying batch buffer\n");
862 return -EINVAL;
863 }
864
865 if (range_overflows_t(u64,
866 eb->batch_start_offset, eb->batch_len,
867 eb->batch->vma->size)) {
868 drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n");
869 return -EINVAL;
870 }
871
872 if (eb->batch_len == 0)
873 eb->batch_len = eb->batch->vma->size - eb->batch_start_offset;
874 if (unlikely(eb->batch_len == 0)) {
875 drm_dbg(&i915->drm, "Invalid batch length\n");
876 return -EINVAL;
877 }
878
879 return 0;
880
881err:
882 eb->vma[i].vma = NULL;
883 return err;
884}
885
886static int eb_validate_vmas(struct i915_execbuffer *eb)
887{
888 unsigned int i;
889 int err;
890
891 INIT_LIST_HEAD(&eb->unbound);
892
893 for (i = 0; i < eb->buffer_count; i++) {
894 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
895 struct eb_vma *ev = &eb->vma[i];
896 struct i915_vma *vma = ev->vma;
897
898 err = i915_gem_object_lock(vma->obj, &eb->ww);
899 if (err)
900 return err;
901
902 if (eb_pin_vma(eb, entry, ev)) {
903 if (entry->offset != vma->node.start) {
904 entry->offset = vma->node.start | UPDATE;
905 eb->args->flags |= __EXEC_HAS_RELOC;
906 }
907 } else {
908 eb_unreserve_vma(ev);
909
910 list_add_tail(&ev->bind_link, &eb->unbound);
911 if (drm_mm_node_allocated(&vma->node)) {
912 err = i915_vma_unbind(vma);
913 if (err)
914 return err;
915 }
916 }
917
918 GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
919 eb_vma_misplaced(&eb->exec[i], vma, ev->flags));
920 }
921
922 if (!list_empty(&eb->unbound))
923 return eb_reserve(eb);
924
925 return 0;
926}
927
928static struct eb_vma *
929eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
930{
931 if (eb->lut_size < 0) {
932 if (handle >= -eb->lut_size)
933 return NULL;
934 return &eb->vma[handle];
935 } else {
936 struct hlist_head *head;
937 struct eb_vma *ev;
938
939 head = &eb->buckets[hash_32(handle, eb->lut_size)];
940 hlist_for_each_entry(ev, head, node) {
941 if (ev->handle == handle)
942 return ev;
943 }
944 return NULL;
945 }
946}
947
948static void eb_release_vmas(struct i915_execbuffer *eb, bool final)
949{
950 const unsigned int count = eb->buffer_count;
951 unsigned int i;
952
953 for (i = 0; i < count; i++) {
954 struct eb_vma *ev = &eb->vma[i];
955 struct i915_vma *vma = ev->vma;
956
957 if (!vma)
958 break;
959
960 eb_unreserve_vma(ev);
961
962 if (final)
963 i915_vma_put(vma);
964 }
965
966 eb_unpin_engine(eb);
967}
968
969static void eb_destroy(const struct i915_execbuffer *eb)
970{
971 GEM_BUG_ON(eb->reloc_cache.rq);
972
973 if (eb->lut_size > 0)
974 kfree(eb->buckets);
975}
976
977static inline u64
978relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
979 const struct i915_vma *target)
980{
981 return gen8_canonical_addr((int)reloc->delta + target->node.start);
982}
983
984static void reloc_cache_clear(struct reloc_cache *cache)
985{
986 cache->rq = NULL;
987 cache->rq_cmd = NULL;
988 cache->pool = NULL;
989 cache->rq_size = 0;
990}
991
992static void reloc_cache_init(struct reloc_cache *cache,
993 struct drm_i915_private *i915)
994{
995 cache->page = -1;
996 cache->vaddr = 0;
997
998 cache->gen = INTEL_GEN(i915);
999 cache->has_llc = HAS_LLC(i915);
1000 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
1001 cache->has_fence = cache->gen < 4;
1002 cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
1003 cache->node.flags = 0;
1004 reloc_cache_clear(cache);
1005}
1006
1007static inline void *unmask_page(unsigned long p)
1008{
1009 return (void *)(uintptr_t)(p & PAGE_MASK);
1010}
1011
1012static inline unsigned int unmask_flags(unsigned long p)
1013{
1014 return p & ~PAGE_MASK;
1015}
1016
1017#define KMAP 0x4
1018
1019static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
1020{
1021 struct drm_i915_private *i915 =
1022 container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
1023 return &i915->ggtt;
1024}
1025
1026static void reloc_cache_put_pool(struct i915_execbuffer *eb, struct reloc_cache *cache)
1027{
1028 if (!cache->pool)
1029 return;
1030
1031
1032
1033
1034
1035
1036
1037 i915_gem_ww_unlock_single(cache->pool->obj);
1038 intel_gt_buffer_pool_put(cache->pool);
1039 cache->pool = NULL;
1040}
1041
1042static void reloc_gpu_flush(struct i915_execbuffer *eb, struct reloc_cache *cache)
1043{
1044 struct drm_i915_gem_object *obj = cache->rq->batch->obj;
1045
1046 GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32));
1047 cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;
1048
1049 __i915_gem_object_flush_map(obj, 0, sizeof(u32) * (cache->rq_size + 1));
1050 i915_gem_object_unpin_map(obj);
1051
1052 intel_gt_chipset_flush(cache->rq->engine->gt);
1053
1054 i915_request_add(cache->rq);
1055 reloc_cache_put_pool(eb, cache);
1056 reloc_cache_clear(cache);
1057
1058 eb->reloc_pool = NULL;
1059}
1060
1061static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb)
1062{
1063 void *vaddr;
1064
1065 if (cache->rq)
1066 reloc_gpu_flush(eb, cache);
1067
1068 if (!cache->vaddr)
1069 return;
1070
1071 vaddr = unmask_page(cache->vaddr);
1072 if (cache->vaddr & KMAP) {
1073 struct drm_i915_gem_object *obj =
1074 (struct drm_i915_gem_object *)cache->node.mm;
1075 if (cache->vaddr & CLFLUSH_AFTER)
1076 mb();
1077
1078 kunmap_atomic(vaddr);
1079 i915_gem_object_finish_access(obj);
1080 } else {
1081 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1082
1083 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1084 io_mapping_unmap_atomic((void __iomem *)vaddr);
1085
1086 if (drm_mm_node_allocated(&cache->node)) {
1087 ggtt->vm.clear_range(&ggtt->vm,
1088 cache->node.start,
1089 cache->node.size);
1090 mutex_lock(&ggtt->vm.mutex);
1091 drm_mm_remove_node(&cache->node);
1092 mutex_unlock(&ggtt->vm.mutex);
1093 } else {
1094 i915_vma_unpin((struct i915_vma *)cache->node.mm);
1095 }
1096 }
1097
1098 cache->vaddr = 0;
1099 cache->page = -1;
1100}
1101
1102static void *reloc_kmap(struct drm_i915_gem_object *obj,
1103 struct reloc_cache *cache,
1104 unsigned long pageno)
1105{
1106 void *vaddr;
1107 struct page *page;
1108
1109 if (cache->vaddr) {
1110 kunmap_atomic(unmask_page(cache->vaddr));
1111 } else {
1112 unsigned int flushes;
1113 int err;
1114
1115 err = i915_gem_object_prepare_write(obj, &flushes);
1116 if (err)
1117 return ERR_PTR(err);
1118
1119 BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
1120 BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
1121
1122 cache->vaddr = flushes | KMAP;
1123 cache->node.mm = (void *)obj;
1124 if (flushes)
1125 mb();
1126 }
1127
1128 page = i915_gem_object_get_page(obj, pageno);
1129 if (!obj->mm.dirty)
1130 set_page_dirty(page);
1131
1132 vaddr = kmap_atomic(page);
1133 cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
1134 cache->page = pageno;
1135
1136 return vaddr;
1137}
1138
1139static void *reloc_iomap(struct drm_i915_gem_object *obj,
1140 struct i915_execbuffer *eb,
1141 unsigned long page)
1142{
1143 struct reloc_cache *cache = &eb->reloc_cache;
1144 struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1145 unsigned long offset;
1146 void *vaddr;
1147
1148 if (cache->vaddr) {
1149 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1150 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
1151 } else {
1152 struct i915_vma *vma;
1153 int err;
1154
1155 if (i915_gem_object_is_tiled(obj))
1156 return ERR_PTR(-EINVAL);
1157
1158 if (use_cpu_reloc(cache, obj))
1159 return NULL;
1160
1161 err = i915_gem_object_set_to_gtt_domain(obj, true);
1162 if (err)
1163 return ERR_PTR(err);
1164
1165 vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0,
1166 PIN_MAPPABLE |
1167 PIN_NONBLOCK |
1168 PIN_NOEVICT);
1169 if (vma == ERR_PTR(-EDEADLK))
1170 return vma;
1171
1172 if (IS_ERR(vma)) {
1173 memset(&cache->node, 0, sizeof(cache->node));
1174 mutex_lock(&ggtt->vm.mutex);
1175 err = drm_mm_insert_node_in_range
1176 (&ggtt->vm.mm, &cache->node,
1177 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
1178 0, ggtt->mappable_end,
1179 DRM_MM_INSERT_LOW);
1180 mutex_unlock(&ggtt->vm.mutex);
1181 if (err)
1182 return NULL;
1183 } else {
1184 cache->node.start = vma->node.start;
1185 cache->node.mm = (void *)vma;
1186 }
1187 }
1188
1189 offset = cache->node.start;
1190 if (drm_mm_node_allocated(&cache->node)) {
1191 ggtt->vm.insert_page(&ggtt->vm,
1192 i915_gem_object_get_dma_address(obj, page),
1193 offset, I915_CACHE_NONE, 0);
1194 } else {
1195 offset += page << PAGE_SHIFT;
1196 }
1197
1198 vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap,
1199 offset);
1200 cache->page = page;
1201 cache->vaddr = (unsigned long)vaddr;
1202
1203 return vaddr;
1204}
1205
1206static void *reloc_vaddr(struct drm_i915_gem_object *obj,
1207 struct i915_execbuffer *eb,
1208 unsigned long page)
1209{
1210 struct reloc_cache *cache = &eb->reloc_cache;
1211 void *vaddr;
1212
1213 if (cache->page == page) {
1214 vaddr = unmask_page(cache->vaddr);
1215 } else {
1216 vaddr = NULL;
1217 if ((cache->vaddr & KMAP) == 0)
1218 vaddr = reloc_iomap(obj, eb, page);
1219 if (!vaddr)
1220 vaddr = reloc_kmap(obj, cache, page);
1221 }
1222
1223 return vaddr;
1224}
1225
1226static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
1227{
1228 if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
1229 if (flushes & CLFLUSH_BEFORE) {
1230 clflushopt(addr);
1231 mb();
1232 }
1233
1234 *addr = value;
1235
1236
1237
1238
1239
1240
1241
1242
1243 if (flushes & CLFLUSH_AFTER)
1244 clflushopt(addr);
1245 } else
1246 *addr = value;
1247}
1248
1249static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma)
1250{
1251 struct drm_i915_gem_object *obj = vma->obj;
1252 int err;
1253
1254 assert_vma_held(vma);
1255
1256 if (obj->cache_dirty & ~obj->cache_coherent)
1257 i915_gem_clflush_object(obj, 0);
1258 obj->write_domain = 0;
1259
1260 err = i915_request_await_object(rq, vma->obj, true);
1261 if (err == 0)
1262 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1263
1264 return err;
1265}
1266
1267static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
1268 struct intel_engine_cs *engine,
1269 struct i915_vma *vma,
1270 unsigned int len)
1271{
1272 struct reloc_cache *cache = &eb->reloc_cache;
1273 struct intel_gt_buffer_pool_node *pool = eb->reloc_pool;
1274 struct i915_request *rq;
1275 struct i915_vma *batch;
1276 u32 *cmd;
1277 int err;
1278
1279 if (!pool) {
1280 pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE);
1281 if (IS_ERR(pool))
1282 return PTR_ERR(pool);
1283 }
1284 eb->reloc_pool = NULL;
1285
1286 err = i915_gem_object_lock(pool->obj, &eb->ww);
1287 if (err)
1288 goto err_pool;
1289
1290 cmd = i915_gem_object_pin_map(pool->obj,
1291 cache->has_llc ?
1292 I915_MAP_FORCE_WB :
1293 I915_MAP_FORCE_WC);
1294 if (IS_ERR(cmd)) {
1295 err = PTR_ERR(cmd);
1296 goto err_pool;
1297 }
1298
1299 batch = i915_vma_instance(pool->obj, vma->vm, NULL);
1300 if (IS_ERR(batch)) {
1301 err = PTR_ERR(batch);
1302 goto err_unmap;
1303 }
1304
1305 err = i915_vma_pin_ww(batch, &eb->ww, 0, 0, PIN_USER | PIN_NONBLOCK);
1306 if (err)
1307 goto err_unmap;
1308
1309 if (engine == eb->context->engine) {
1310 rq = i915_request_create(eb->context);
1311 } else {
1312 struct intel_context *ce = eb->reloc_context;
1313
1314 if (!ce) {
1315 ce = intel_context_create(engine);
1316 if (IS_ERR(ce)) {
1317 err = PTR_ERR(ce);
1318 goto err_unpin;
1319 }
1320
1321 i915_vm_put(ce->vm);
1322 ce->vm = i915_vm_get(eb->context->vm);
1323 eb->reloc_context = ce;
1324 }
1325
1326 err = intel_context_pin_ww(ce, &eb->ww);
1327 if (err)
1328 goto err_unpin;
1329
1330 rq = i915_request_create(ce);
1331 intel_context_unpin(ce);
1332 }
1333 if (IS_ERR(rq)) {
1334 err = PTR_ERR(rq);
1335 goto err_unpin;
1336 }
1337
1338 err = intel_gt_buffer_pool_mark_active(pool, rq);
1339 if (err)
1340 goto err_request;
1341
1342 err = reloc_move_to_gpu(rq, vma);
1343 if (err)
1344 goto err_request;
1345
1346 err = eb->engine->emit_bb_start(rq,
1347 batch->node.start, PAGE_SIZE,
1348 cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
1349 if (err)
1350 goto skip_request;
1351
1352 assert_vma_held(batch);
1353 err = i915_request_await_object(rq, batch->obj, false);
1354 if (err == 0)
1355 err = i915_vma_move_to_active(batch, rq, 0);
1356 if (err)
1357 goto skip_request;
1358
1359 rq->batch = batch;
1360 i915_vma_unpin(batch);
1361
1362 cache->rq = rq;
1363 cache->rq_cmd = cmd;
1364 cache->rq_size = 0;
1365 cache->pool = pool;
1366
1367
1368 return 0;
1369
1370skip_request:
1371 i915_request_set_error_once(rq, err);
1372err_request:
1373 i915_request_add(rq);
1374err_unpin:
1375 i915_vma_unpin(batch);
1376err_unmap:
1377 i915_gem_object_unpin_map(pool->obj);
1378err_pool:
1379 eb->reloc_pool = pool;
1380 return err;
1381}
1382
1383static bool reloc_can_use_engine(const struct intel_engine_cs *engine)
1384{
1385 return engine->class != VIDEO_DECODE_CLASS || !IS_GEN(engine->i915, 6);
1386}
1387
1388static u32 *reloc_gpu(struct i915_execbuffer *eb,
1389 struct i915_vma *vma,
1390 unsigned int len)
1391{
1392 struct reloc_cache *cache = &eb->reloc_cache;
1393 u32 *cmd;
1394
1395 if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1))
1396 reloc_gpu_flush(eb, cache);
1397
1398 if (unlikely(!cache->rq)) {
1399 int err;
1400 struct intel_engine_cs *engine = eb->engine;
1401
1402 if (!reloc_can_use_engine(engine)) {
1403 engine = engine->gt->engine_class[COPY_ENGINE_CLASS][0];
1404 if (!engine)
1405 return ERR_PTR(-ENODEV);
1406 }
1407
1408 err = __reloc_gpu_alloc(eb, engine, vma, len);
1409 if (unlikely(err))
1410 return ERR_PTR(err);
1411 }
1412
1413 cmd = cache->rq_cmd + cache->rq_size;
1414 cache->rq_size += len;
1415
1416 return cmd;
1417}
1418
1419static inline bool use_reloc_gpu(struct i915_vma *vma)
1420{
1421 if (DBG_FORCE_RELOC == FORCE_GPU_RELOC)
1422 return true;
1423
1424 if (DBG_FORCE_RELOC)
1425 return false;
1426
1427 return !dma_resv_test_signaled_rcu(vma->resv, true);
1428}
1429
1430static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset)
1431{
1432 struct page *page;
1433 unsigned long addr;
1434
1435 GEM_BUG_ON(vma->pages != vma->obj->mm.pages);
1436
1437 page = i915_gem_object_get_page(vma->obj, offset >> PAGE_SHIFT);
1438 addr = PFN_PHYS(page_to_pfn(page));
1439 GEM_BUG_ON(overflows_type(addr, u32));
1440
1441 return addr + offset_in_page(offset);
1442}
1443
1444static int __reloc_entry_gpu(struct i915_execbuffer *eb,
1445 struct i915_vma *vma,
1446 u64 offset,
1447 u64 target_addr)
1448{
1449 const unsigned int gen = eb->reloc_cache.gen;
1450 unsigned int len;
1451 u32 *batch;
1452 u64 addr;
1453
1454 if (gen >= 8)
1455 len = offset & 7 ? 8 : 5;
1456 else if (gen >= 4)
1457 len = 4;
1458 else
1459 len = 3;
1460
1461 batch = reloc_gpu(eb, vma, len);
1462 if (batch == ERR_PTR(-EDEADLK))
1463 return -EDEADLK;
1464 else if (IS_ERR(batch))
1465 return false;
1466
1467 addr = gen8_canonical_addr(vma->node.start + offset);
1468 if (gen >= 8) {
1469 if (offset & 7) {
1470 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1471 *batch++ = lower_32_bits(addr);
1472 *batch++ = upper_32_bits(addr);
1473 *batch++ = lower_32_bits(target_addr);
1474
1475 addr = gen8_canonical_addr(addr + 4);
1476
1477 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1478 *batch++ = lower_32_bits(addr);
1479 *batch++ = upper_32_bits(addr);
1480 *batch++ = upper_32_bits(target_addr);
1481 } else {
1482 *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1;
1483 *batch++ = lower_32_bits(addr);
1484 *batch++ = upper_32_bits(addr);
1485 *batch++ = lower_32_bits(target_addr);
1486 *batch++ = upper_32_bits(target_addr);
1487 }
1488 } else if (gen >= 6) {
1489 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1490 *batch++ = 0;
1491 *batch++ = addr;
1492 *batch++ = target_addr;
1493 } else if (IS_I965G(eb->i915)) {
1494 *batch++ = MI_STORE_DWORD_IMM_GEN4;
1495 *batch++ = 0;
1496 *batch++ = vma_phys_addr(vma, offset);
1497 *batch++ = target_addr;
1498 } else if (gen >= 4) {
1499 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1500 *batch++ = 0;
1501 *batch++ = addr;
1502 *batch++ = target_addr;
1503 } else if (gen >= 3 &&
1504 !(IS_I915G(eb->i915) || IS_I915GM(eb->i915))) {
1505 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
1506 *batch++ = addr;
1507 *batch++ = target_addr;
1508 } else {
1509 *batch++ = MI_STORE_DWORD_IMM;
1510 *batch++ = vma_phys_addr(vma, offset);
1511 *batch++ = target_addr;
1512 }
1513
1514 return true;
1515}
1516
1517static int reloc_entry_gpu(struct i915_execbuffer *eb,
1518 struct i915_vma *vma,
1519 u64 offset,
1520 u64 target_addr)
1521{
1522 if (eb->reloc_cache.vaddr)
1523 return false;
1524
1525 if (!use_reloc_gpu(vma))
1526 return false;
1527
1528 return __reloc_entry_gpu(eb, vma, offset, target_addr);
1529}
1530
1531static u64
1532relocate_entry(struct i915_vma *vma,
1533 const struct drm_i915_gem_relocation_entry *reloc,
1534 struct i915_execbuffer *eb,
1535 const struct i915_vma *target)
1536{
1537 u64 target_addr = relocation_target(reloc, target);
1538 u64 offset = reloc->offset;
1539 int reloc_gpu = reloc_entry_gpu(eb, vma, offset, target_addr);
1540
1541 if (reloc_gpu < 0)
1542 return reloc_gpu;
1543
1544 if (!reloc_gpu) {
1545 bool wide = eb->reloc_cache.use_64bit_reloc;
1546 void *vaddr;
1547
1548repeat:
1549 vaddr = reloc_vaddr(vma->obj, eb,
1550 offset >> PAGE_SHIFT);
1551 if (IS_ERR(vaddr))
1552 return PTR_ERR(vaddr);
1553
1554 GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32)));
1555 clflush_write32(vaddr + offset_in_page(offset),
1556 lower_32_bits(target_addr),
1557 eb->reloc_cache.vaddr);
1558
1559 if (wide) {
1560 offset += sizeof(u32);
1561 target_addr >>= 32;
1562 wide = false;
1563 goto repeat;
1564 }
1565 }
1566
1567 return target->node.start | UPDATE;
1568}
1569
1570static u64
1571eb_relocate_entry(struct i915_execbuffer *eb,
1572 struct eb_vma *ev,
1573 const struct drm_i915_gem_relocation_entry *reloc)
1574{
1575 struct drm_i915_private *i915 = eb->i915;
1576 struct eb_vma *target;
1577 int err;
1578
1579
1580 target = eb_get_vma(eb, reloc->target_handle);
1581 if (unlikely(!target))
1582 return -ENOENT;
1583
1584
1585 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
1586 drm_dbg(&i915->drm, "reloc with multiple write domains: "
1587 "target %d offset %d "
1588 "read %08x write %08x",
1589 reloc->target_handle,
1590 (int) reloc->offset,
1591 reloc->read_domains,
1592 reloc->write_domain);
1593 return -EINVAL;
1594 }
1595 if (unlikely((reloc->write_domain | reloc->read_domains)
1596 & ~I915_GEM_GPU_DOMAINS)) {
1597 drm_dbg(&i915->drm, "reloc with read/write non-GPU domains: "
1598 "target %d offset %d "
1599 "read %08x write %08x",
1600 reloc->target_handle,
1601 (int) reloc->offset,
1602 reloc->read_domains,
1603 reloc->write_domain);
1604 return -EINVAL;
1605 }
1606
1607 if (reloc->write_domain) {
1608 target->flags |= EXEC_OBJECT_WRITE;
1609
1610
1611
1612
1613
1614
1615
1616 if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
1617 IS_GEN(eb->i915, 6)) {
1618 err = i915_vma_bind(target->vma,
1619 target->vma->obj->cache_level,
1620 PIN_GLOBAL, NULL);
1621 if (err)
1622 return err;
1623 }
1624 }
1625
1626
1627
1628
1629
1630 if (!DBG_FORCE_RELOC &&
1631 gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset)
1632 return 0;
1633
1634
1635 if (unlikely(reloc->offset >
1636 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
1637 drm_dbg(&i915->drm, "Relocation beyond object bounds: "
1638 "target %d offset %d size %d.\n",
1639 reloc->target_handle,
1640 (int)reloc->offset,
1641 (int)ev->vma->size);
1642 return -EINVAL;
1643 }
1644 if (unlikely(reloc->offset & 3)) {
1645 drm_dbg(&i915->drm, "Relocation not 4-byte aligned: "
1646 "target %d offset %d.\n",
1647 reloc->target_handle,
1648 (int)reloc->offset);
1649 return -EINVAL;
1650 }
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660 ev->flags &= ~EXEC_OBJECT_ASYNC;
1661
1662
1663 return relocate_entry(ev->vma, reloc, eb, target->vma);
1664}
1665
1666static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev)
1667{
1668#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
1669 struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
1670 const struct drm_i915_gem_exec_object2 *entry = ev->exec;
1671 struct drm_i915_gem_relocation_entry __user *urelocs =
1672 u64_to_user_ptr(entry->relocs_ptr);
1673 unsigned long remain = entry->relocation_count;
1674
1675 if (unlikely(remain > N_RELOC(ULONG_MAX)))
1676 return -EINVAL;
1677
1678
1679
1680
1681
1682
1683 if (unlikely(!access_ok(urelocs, remain * sizeof(*urelocs))))
1684 return -EFAULT;
1685
1686 do {
1687 struct drm_i915_gem_relocation_entry *r = stack;
1688 unsigned int count =
1689 min_t(unsigned long, remain, ARRAY_SIZE(stack));
1690 unsigned int copied;
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700 pagefault_disable();
1701 copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
1702 pagefault_enable();
1703 if (unlikely(copied)) {
1704 remain = -EFAULT;
1705 goto out;
1706 }
1707
1708 remain -= count;
1709 do {
1710 u64 offset = eb_relocate_entry(eb, ev, r);
1711
1712 if (likely(offset == 0)) {
1713 } else if ((s64)offset < 0) {
1714 remain = (int)offset;
1715 goto out;
1716 } else {
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738 offset = gen8_canonical_addr(offset & ~UPDATE);
1739 __put_user(offset,
1740 &urelocs[r - stack].presumed_offset);
1741 }
1742 } while (r++, --count);
1743 urelocs += ARRAY_SIZE(stack);
1744 } while (remain);
1745out:
1746 reloc_cache_reset(&eb->reloc_cache, eb);
1747 return remain;
1748}
1749
1750static int
1751eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev)
1752{
1753 const struct drm_i915_gem_exec_object2 *entry = ev->exec;
1754 struct drm_i915_gem_relocation_entry *relocs =
1755 u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1756 unsigned int i;
1757 int err;
1758
1759 for (i = 0; i < entry->relocation_count; i++) {
1760 u64 offset = eb_relocate_entry(eb, ev, &relocs[i]);
1761
1762 if ((s64)offset < 0) {
1763 err = (int)offset;
1764 goto err;
1765 }
1766 }
1767 err = 0;
1768err:
1769 reloc_cache_reset(&eb->reloc_cache, eb);
1770 return err;
1771}
1772
1773static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
1774{
1775 const char __user *addr, *end;
1776 unsigned long size;
1777 char __maybe_unused c;
1778
1779 size = entry->relocation_count;
1780 if (size == 0)
1781 return 0;
1782
1783 if (size > N_RELOC(ULONG_MAX))
1784 return -EINVAL;
1785
1786 addr = u64_to_user_ptr(entry->relocs_ptr);
1787 size *= sizeof(struct drm_i915_gem_relocation_entry);
1788 if (!access_ok(addr, size))
1789 return -EFAULT;
1790
1791 end = addr + size;
1792 for (; addr < end; addr += PAGE_SIZE) {
1793 int err = __get_user(c, addr);
1794 if (err)
1795 return err;
1796 }
1797 return __get_user(c, end - 1);
1798}
1799
1800static int eb_copy_relocations(const struct i915_execbuffer *eb)
1801{
1802 struct drm_i915_gem_relocation_entry *relocs;
1803 const unsigned int count = eb->buffer_count;
1804 unsigned int i;
1805 int err;
1806
1807 for (i = 0; i < count; i++) {
1808 const unsigned int nreloc = eb->exec[i].relocation_count;
1809 struct drm_i915_gem_relocation_entry __user *urelocs;
1810 unsigned long size;
1811 unsigned long copied;
1812
1813 if (nreloc == 0)
1814 continue;
1815
1816 err = check_relocations(&eb->exec[i]);
1817 if (err)
1818 goto err;
1819
1820 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
1821 size = nreloc * sizeof(*relocs);
1822
1823 relocs = kvmalloc_array(size, 1, GFP_KERNEL);
1824 if (!relocs) {
1825 err = -ENOMEM;
1826 goto err;
1827 }
1828
1829
1830 copied = 0;
1831 do {
1832 unsigned int len =
1833 min_t(u64, BIT_ULL(31), size - copied);
1834
1835 if (__copy_from_user((char *)relocs + copied,
1836 (char __user *)urelocs + copied,
1837 len))
1838 goto end;
1839
1840 copied += len;
1841 } while (copied < size);
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853 if (!user_access_begin(urelocs, size))
1854 goto end;
1855
1856 for (copied = 0; copied < nreloc; copied++)
1857 unsafe_put_user(-1,
1858 &urelocs[copied].presumed_offset,
1859 end_user);
1860 user_access_end();
1861
1862 eb->exec[i].relocs_ptr = (uintptr_t)relocs;
1863 }
1864
1865 return 0;
1866
1867end_user:
1868 user_access_end();
1869end:
1870 kvfree(relocs);
1871 err = -EFAULT;
1872err:
1873 while (i--) {
1874 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
1875 if (eb->exec[i].relocation_count)
1876 kvfree(relocs);
1877 }
1878 return err;
1879}
1880
1881static int eb_prefault_relocations(const struct i915_execbuffer *eb)
1882{
1883 const unsigned int count = eb->buffer_count;
1884 unsigned int i;
1885
1886 for (i = 0; i < count; i++) {
1887 int err;
1888
1889 err = check_relocations(&eb->exec[i]);
1890 if (err)
1891 return err;
1892 }
1893
1894 return 0;
1895}
1896
1897static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb,
1898 struct i915_request *rq)
1899{
1900 bool have_copy = false;
1901 struct eb_vma *ev;
1902 int err = 0;
1903
1904repeat:
1905 if (signal_pending(current)) {
1906 err = -ERESTARTSYS;
1907 goto out;
1908 }
1909
1910
1911 eb_release_vmas(eb, false);
1912 i915_gem_ww_ctx_fini(&eb->ww);
1913
1914 if (rq) {
1915
1916 if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE,
1917 MAX_SCHEDULE_TIMEOUT) < 0) {
1918 i915_request_put(rq);
1919 rq = NULL;
1920
1921 err = -EINTR;
1922 goto err_relock;
1923 }
1924
1925 i915_request_put(rq);
1926 rq = NULL;
1927 }
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942 if (!err) {
1943 err = eb_prefault_relocations(eb);
1944 } else if (!have_copy) {
1945 err = eb_copy_relocations(eb);
1946 have_copy = err == 0;
1947 } else {
1948 cond_resched();
1949 err = 0;
1950 }
1951
1952 if (!err)
1953 flush_workqueue(eb->i915->mm.userptr_wq);
1954
1955err_relock:
1956 i915_gem_ww_ctx_init(&eb->ww, true);
1957 if (err)
1958 goto out;
1959
1960
1961repeat_validate:
1962 rq = eb_pin_engine(eb, false);
1963 if (IS_ERR(rq)) {
1964 err = PTR_ERR(rq);
1965 rq = NULL;
1966 goto err;
1967 }
1968
1969
1970 GEM_WARN_ON(rq);
1971
1972 err = eb_validate_vmas(eb);
1973 if (err)
1974 goto err;
1975
1976 GEM_BUG_ON(!eb->batch);
1977
1978 list_for_each_entry(ev, &eb->relocs, reloc_link) {
1979 if (!have_copy) {
1980 pagefault_disable();
1981 err = eb_relocate_vma(eb, ev);
1982 pagefault_enable();
1983 if (err)
1984 break;
1985 } else {
1986 err = eb_relocate_vma_slow(eb, ev);
1987 if (err)
1988 break;
1989 }
1990 }
1991
1992 if (err == -EDEADLK)
1993 goto err;
1994
1995 if (err && !have_copy)
1996 goto repeat;
1997
1998 if (err)
1999 goto err;
2000
2001
2002 err = eb_parse(eb);
2003 if (err)
2004 goto err;
2005
2006
2007
2008
2009
2010
2011
2012
2013err:
2014 if (err == -EDEADLK) {
2015 eb_release_vmas(eb, false);
2016 err = i915_gem_ww_ctx_backoff(&eb->ww);
2017 if (!err)
2018 goto repeat_validate;
2019 }
2020
2021 if (err == -EAGAIN)
2022 goto repeat;
2023
2024out:
2025 if (have_copy) {
2026 const unsigned int count = eb->buffer_count;
2027 unsigned int i;
2028
2029 for (i = 0; i < count; i++) {
2030 const struct drm_i915_gem_exec_object2 *entry =
2031 &eb->exec[i];
2032 struct drm_i915_gem_relocation_entry *relocs;
2033
2034 if (!entry->relocation_count)
2035 continue;
2036
2037 relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
2038 kvfree(relocs);
2039 }
2040 }
2041
2042 if (rq)
2043 i915_request_put(rq);
2044
2045 return err;
2046}
2047
2048static int eb_relocate_parse(struct i915_execbuffer *eb)
2049{
2050 int err;
2051 struct i915_request *rq = NULL;
2052 bool throttle = true;
2053
2054retry:
2055 rq = eb_pin_engine(eb, throttle);
2056 if (IS_ERR(rq)) {
2057 err = PTR_ERR(rq);
2058 rq = NULL;
2059 if (err != -EDEADLK)
2060 return err;
2061
2062 goto err;
2063 }
2064
2065 if (rq) {
2066 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK;
2067
2068
2069 err = i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, 0);
2070 if (err == -ETIME) {
2071 if (nonblock) {
2072 err = -EWOULDBLOCK;
2073 i915_request_put(rq);
2074 goto err;
2075 }
2076 goto slow;
2077 }
2078 i915_request_put(rq);
2079 rq = NULL;
2080 }
2081
2082
2083 throttle = false;
2084
2085 err = eb_validate_vmas(eb);
2086 if (err == -EAGAIN)
2087 goto slow;
2088 else if (err)
2089 goto err;
2090
2091
2092 if (eb->args->flags & __EXEC_HAS_RELOC) {
2093 struct eb_vma *ev;
2094
2095 list_for_each_entry(ev, &eb->relocs, reloc_link) {
2096 err = eb_relocate_vma(eb, ev);
2097 if (err)
2098 break;
2099 }
2100
2101 if (err == -EDEADLK)
2102 goto err;
2103 else if (err)
2104 goto slow;
2105 }
2106
2107 if (!err)
2108 err = eb_parse(eb);
2109
2110err:
2111 if (err == -EDEADLK) {
2112 eb_release_vmas(eb, false);
2113 err = i915_gem_ww_ctx_backoff(&eb->ww);
2114 if (!err)
2115 goto retry;
2116 }
2117
2118 return err;
2119
2120slow:
2121 err = eb_relocate_parse_slow(eb, rq);
2122 if (err)
2123
2124
2125
2126
2127
2128
2129
2130 eb->args->flags &= ~__EXEC_HAS_RELOC;
2131
2132 return err;
2133}
2134
2135static int eb_move_to_gpu(struct i915_execbuffer *eb)
2136{
2137 const unsigned int count = eb->buffer_count;
2138 unsigned int i = count;
2139 int err = 0;
2140
2141 while (i--) {
2142 struct eb_vma *ev = &eb->vma[i];
2143 struct i915_vma *vma = ev->vma;
2144 unsigned int flags = ev->flags;
2145 struct drm_i915_gem_object *obj = vma->obj;
2146
2147 assert_vma_held(vma);
2148
2149 if (flags & EXEC_OBJECT_CAPTURE) {
2150 struct i915_capture_list *capture;
2151
2152 capture = kmalloc(sizeof(*capture), GFP_KERNEL);
2153 if (capture) {
2154 capture->next = eb->request->capture_list;
2155 capture->vma = vma;
2156 eb->request->capture_list = capture;
2157 }
2158 }
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172 if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) {
2173 if (i915_gem_clflush_object(obj, 0))
2174 flags &= ~EXEC_OBJECT_ASYNC;
2175 }
2176
2177 if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) {
2178 err = i915_request_await_object
2179 (eb->request, obj, flags & EXEC_OBJECT_WRITE);
2180 }
2181
2182 if (err == 0)
2183 err = i915_vma_move_to_active(vma, eb->request, flags);
2184 }
2185
2186 if (unlikely(err))
2187 goto err_skip;
2188
2189
2190 intel_gt_chipset_flush(eb->engine->gt);
2191 return 0;
2192
2193err_skip:
2194 i915_request_set_error_once(eb->request, err);
2195 return err;
2196}
2197
2198static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
2199{
2200 if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
2201 return -EINVAL;
2202
2203
2204 if (!(exec->flags & (I915_EXEC_FENCE_ARRAY |
2205 I915_EXEC_USE_EXTENSIONS))) {
2206 if (exec->num_cliprects || exec->cliprects_ptr)
2207 return -EINVAL;
2208 }
2209
2210 if (exec->DR4 == 0xffffffff) {
2211 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
2212 exec->DR4 = 0;
2213 }
2214 if (exec->DR1 || exec->DR4)
2215 return -EINVAL;
2216
2217 if ((exec->batch_start_offset | exec->batch_len) & 0x7)
2218 return -EINVAL;
2219
2220 return 0;
2221}
2222
2223static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
2224{
2225 u32 *cs;
2226 int i;
2227
2228 if (!IS_GEN(rq->engine->i915, 7) || rq->engine->id != RCS0) {
2229 drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n");
2230 return -EINVAL;
2231 }
2232
2233 cs = intel_ring_begin(rq, 4 * 2 + 2);
2234 if (IS_ERR(cs))
2235 return PTR_ERR(cs);
2236
2237 *cs++ = MI_LOAD_REGISTER_IMM(4);
2238 for (i = 0; i < 4; i++) {
2239 *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
2240 *cs++ = 0;
2241 }
2242 *cs++ = MI_NOOP;
2243 intel_ring_advance(rq, cs);
2244
2245 return 0;
2246}
2247
2248static struct i915_vma *
2249shadow_batch_pin(struct i915_execbuffer *eb,
2250 struct drm_i915_gem_object *obj,
2251 struct i915_address_space *vm,
2252 unsigned int flags)
2253{
2254 struct i915_vma *vma;
2255 int err;
2256
2257 vma = i915_vma_instance(obj, vm, NULL);
2258 if (IS_ERR(vma))
2259 return vma;
2260
2261 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags);
2262 if (err)
2263 return ERR_PTR(err);
2264
2265 return vma;
2266}
2267
2268struct eb_parse_work {
2269 struct dma_fence_work base;
2270 struct intel_engine_cs *engine;
2271 struct i915_vma *batch;
2272 struct i915_vma *shadow;
2273 struct i915_vma *trampoline;
2274 unsigned long batch_offset;
2275 unsigned long batch_length;
2276};
2277
2278static int __eb_parse(struct dma_fence_work *work)
2279{
2280 struct eb_parse_work *pw = container_of(work, typeof(*pw), base);
2281
2282 return intel_engine_cmd_parser(pw->engine,
2283 pw->batch,
2284 pw->batch_offset,
2285 pw->batch_length,
2286 pw->shadow,
2287 pw->trampoline);
2288}
2289
2290static void __eb_parse_release(struct dma_fence_work *work)
2291{
2292 struct eb_parse_work *pw = container_of(work, typeof(*pw), base);
2293
2294 if (pw->trampoline)
2295 i915_active_release(&pw->trampoline->active);
2296 i915_active_release(&pw->shadow->active);
2297 i915_active_release(&pw->batch->active);
2298}
2299
2300static const struct dma_fence_work_ops eb_parse_ops = {
2301 .name = "eb_parse",
2302 .work = __eb_parse,
2303 .release = __eb_parse_release,
2304};
2305
2306static inline int
2307__parser_mark_active(struct i915_vma *vma,
2308 struct intel_timeline *tl,
2309 struct dma_fence *fence)
2310{
2311 struct intel_gt_buffer_pool_node *node = vma->private;
2312
2313 return i915_active_ref(&node->active, tl->fence_context, fence);
2314}
2315
2316static int
2317parser_mark_active(struct eb_parse_work *pw, struct intel_timeline *tl)
2318{
2319 int err;
2320
2321 mutex_lock(&tl->mutex);
2322
2323 err = __parser_mark_active(pw->shadow, tl, &pw->base.dma);
2324 if (err)
2325 goto unlock;
2326
2327 if (pw->trampoline) {
2328 err = __parser_mark_active(pw->trampoline, tl, &pw->base.dma);
2329 if (err)
2330 goto unlock;
2331 }
2332
2333unlock:
2334 mutex_unlock(&tl->mutex);
2335 return err;
2336}
2337
2338static int eb_parse_pipeline(struct i915_execbuffer *eb,
2339 struct i915_vma *shadow,
2340 struct i915_vma *trampoline)
2341{
2342 struct eb_parse_work *pw;
2343 int err;
2344
2345 GEM_BUG_ON(overflows_type(eb->batch_start_offset, pw->batch_offset));
2346 GEM_BUG_ON(overflows_type(eb->batch_len, pw->batch_length));
2347
2348 pw = kzalloc(sizeof(*pw), GFP_KERNEL);
2349 if (!pw)
2350 return -ENOMEM;
2351
2352 err = i915_active_acquire(&eb->batch->vma->active);
2353 if (err)
2354 goto err_free;
2355
2356 err = i915_active_acquire(&shadow->active);
2357 if (err)
2358 goto err_batch;
2359
2360 if (trampoline) {
2361 err = i915_active_acquire(&trampoline->active);
2362 if (err)
2363 goto err_shadow;
2364 }
2365
2366 dma_fence_work_init(&pw->base, &eb_parse_ops);
2367
2368 pw->engine = eb->engine;
2369 pw->batch = eb->batch->vma;
2370 pw->batch_offset = eb->batch_start_offset;
2371 pw->batch_length = eb->batch_len;
2372 pw->shadow = shadow;
2373 pw->trampoline = trampoline;
2374
2375
2376 err = parser_mark_active(pw, eb->context->timeline);
2377 if (err)
2378 goto err_commit;
2379
2380 err = dma_resv_reserve_shared(pw->batch->resv, 1);
2381 if (err)
2382 goto err_commit;
2383
2384
2385 err = i915_sw_fence_await_reservation(&pw->base.chain,
2386 pw->batch->resv, NULL, false,
2387 0, I915_FENCE_GFP);
2388 if (err < 0)
2389 goto err_commit;
2390
2391
2392 dma_resv_add_shared_fence(pw->batch->resv, &pw->base.dma);
2393
2394
2395 dma_resv_add_excl_fence(shadow->resv, &pw->base.dma);
2396
2397 dma_fence_work_commit_imm(&pw->base);
2398 return 0;
2399
2400err_commit:
2401 i915_sw_fence_set_error_once(&pw->base.chain, err);
2402 dma_fence_work_commit_imm(&pw->base);
2403 return err;
2404
2405err_shadow:
2406 i915_active_release(&shadow->active);
2407err_batch:
2408 i915_active_release(&eb->batch->vma->active);
2409err_free:
2410 kfree(pw);
2411 return err;
2412}
2413
2414static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma)
2415{
2416
2417
2418
2419
2420 if (eb->batch_flags & I915_DISPATCH_SECURE)
2421 return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0);
2422
2423 return NULL;
2424}
2425
2426static int eb_parse(struct i915_execbuffer *eb)
2427{
2428 struct drm_i915_private *i915 = eb->i915;
2429 struct intel_gt_buffer_pool_node *pool = eb->batch_pool;
2430 struct i915_vma *shadow, *trampoline, *batch;
2431 unsigned long len;
2432 int err;
2433
2434 if (!eb_use_cmdparser(eb)) {
2435 batch = eb_dispatch_secure(eb, eb->batch->vma);
2436 if (IS_ERR(batch))
2437 return PTR_ERR(batch);
2438
2439 goto secure_batch;
2440 }
2441
2442 len = eb->batch_len;
2443 if (!CMDPARSER_USES_GGTT(eb->i915)) {
2444
2445
2446
2447
2448 if (!eb->context->vm->has_read_only) {
2449 drm_dbg(&i915->drm,
2450 "Cannot prevent post-scan tampering without RO capable vm\n");
2451 return -EINVAL;
2452 }
2453 } else {
2454 len += I915_CMD_PARSER_TRAMPOLINE_SIZE;
2455 }
2456 if (unlikely(len < eb->batch_len))
2457 return -EINVAL;
2458
2459 if (!pool) {
2460 pool = intel_gt_get_buffer_pool(eb->engine->gt, len);
2461 if (IS_ERR(pool))
2462 return PTR_ERR(pool);
2463 eb->batch_pool = pool;
2464 }
2465
2466 err = i915_gem_object_lock(pool->obj, &eb->ww);
2467 if (err)
2468 goto err;
2469
2470 shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER);
2471 if (IS_ERR(shadow)) {
2472 err = PTR_ERR(shadow);
2473 goto err;
2474 }
2475 i915_gem_object_set_readonly(shadow->obj);
2476 shadow->private = pool;
2477
2478 trampoline = NULL;
2479 if (CMDPARSER_USES_GGTT(eb->i915)) {
2480 trampoline = shadow;
2481
2482 shadow = shadow_batch_pin(eb, pool->obj,
2483 &eb->engine->gt->ggtt->vm,
2484 PIN_GLOBAL);
2485 if (IS_ERR(shadow)) {
2486 err = PTR_ERR(shadow);
2487 shadow = trampoline;
2488 goto err_shadow;
2489 }
2490 shadow->private = pool;
2491
2492 eb->batch_flags |= I915_DISPATCH_SECURE;
2493 }
2494
2495 batch = eb_dispatch_secure(eb, shadow);
2496 if (IS_ERR(batch)) {
2497 err = PTR_ERR(batch);
2498 goto err_trampoline;
2499 }
2500
2501 err = eb_parse_pipeline(eb, shadow, trampoline);
2502 if (err)
2503 goto err_unpin_batch;
2504
2505 eb->batch = &eb->vma[eb->buffer_count++];
2506 eb->batch->vma = i915_vma_get(shadow);
2507 eb->batch->flags = __EXEC_OBJECT_HAS_PIN;
2508
2509 eb->trampoline = trampoline;
2510 eb->batch_start_offset = 0;
2511
2512secure_batch:
2513 if (batch) {
2514 eb->batch = &eb->vma[eb->buffer_count++];
2515 eb->batch->flags = __EXEC_OBJECT_HAS_PIN;
2516 eb->batch->vma = i915_vma_get(batch);
2517 }
2518 return 0;
2519
2520err_unpin_batch:
2521 if (batch)
2522 i915_vma_unpin(batch);
2523err_trampoline:
2524 if (trampoline)
2525 i915_vma_unpin(trampoline);
2526err_shadow:
2527 i915_vma_unpin(shadow);
2528err:
2529 return err;
2530}
2531
2532static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch)
2533{
2534 int err;
2535
2536 err = eb_move_to_gpu(eb);
2537 if (err)
2538 return err;
2539
2540 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
2541 err = i915_reset_gen7_sol_offsets(eb->request);
2542 if (err)
2543 return err;
2544 }
2545
2546
2547
2548
2549
2550
2551
2552 if (eb->engine->emit_init_breadcrumb) {
2553 err = eb->engine->emit_init_breadcrumb(eb->request);
2554 if (err)
2555 return err;
2556 }
2557
2558 err = eb->engine->emit_bb_start(eb->request,
2559 batch->node.start +
2560 eb->batch_start_offset,
2561 eb->batch_len,
2562 eb->batch_flags);
2563 if (err)
2564 return err;
2565
2566 if (eb->trampoline) {
2567 GEM_BUG_ON(eb->batch_start_offset);
2568 err = eb->engine->emit_bb_start(eb->request,
2569 eb->trampoline->node.start +
2570 eb->batch_len,
2571 0, 0);
2572 if (err)
2573 return err;
2574 }
2575
2576 if (intel_context_nopreempt(eb->context))
2577 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &eb->request->fence.flags);
2578
2579 return 0;
2580}
2581
2582static int num_vcs_engines(const struct drm_i915_private *i915)
2583{
2584 return hweight64(VDBOX_MASK(&i915->gt));
2585}
2586
2587
2588
2589
2590
2591static unsigned int
2592gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
2593 struct drm_file *file)
2594{
2595 struct drm_i915_file_private *file_priv = file->driver_priv;
2596
2597
2598 if ((int)file_priv->bsd_engine < 0)
2599 file_priv->bsd_engine =
2600 get_random_int() % num_vcs_engines(dev_priv);
2601
2602 return file_priv->bsd_engine;
2603}
2604
2605static const enum intel_engine_id user_ring_map[] = {
2606 [I915_EXEC_DEFAULT] = RCS0,
2607 [I915_EXEC_RENDER] = RCS0,
2608 [I915_EXEC_BLT] = BCS0,
2609 [I915_EXEC_BSD] = VCS0,
2610 [I915_EXEC_VEBOX] = VECS0
2611};
2612
2613static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce)
2614{
2615 struct intel_ring *ring = ce->ring;
2616 struct intel_timeline *tl = ce->timeline;
2617 struct i915_request *rq;
2618
2619
2620
2621
2622
2623 if (intel_ring_update_space(ring) >= PAGE_SIZE)
2624 return NULL;
2625
2626
2627
2628
2629
2630
2631
2632
2633 list_for_each_entry(rq, &tl->requests, link) {
2634 if (rq->ring != ring)
2635 continue;
2636
2637 if (__intel_ring_space(rq->postfix,
2638 ring->emit, ring->size) > ring->size / 2)
2639 break;
2640 }
2641 if (&rq->link == &tl->requests)
2642 return NULL;
2643
2644 return i915_request_get(rq);
2645}
2646
2647static struct i915_request *eb_pin_engine(struct i915_execbuffer *eb, bool throttle)
2648{
2649 struct intel_context *ce = eb->context;
2650 struct intel_timeline *tl;
2651 struct i915_request *rq = NULL;
2652 int err;
2653
2654 GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED);
2655
2656 if (unlikely(intel_context_is_banned(ce)))
2657 return ERR_PTR(-EIO);
2658
2659
2660
2661
2662
2663
2664 err = intel_context_pin_ww(ce, &eb->ww);
2665 if (err)
2666 return ERR_PTR(err);
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676 tl = intel_context_timeline_lock(ce);
2677 if (IS_ERR(tl)) {
2678 intel_context_unpin(ce);
2679 return ERR_CAST(tl);
2680 }
2681
2682 intel_context_enter(ce);
2683 if (throttle)
2684 rq = eb_throttle(eb, ce);
2685 intel_context_timeline_unlock(tl);
2686
2687 eb->args->flags |= __EXEC_ENGINE_PINNED;
2688 return rq;
2689}
2690
2691static void eb_unpin_engine(struct i915_execbuffer *eb)
2692{
2693 struct intel_context *ce = eb->context;
2694 struct intel_timeline *tl = ce->timeline;
2695
2696 if (!(eb->args->flags & __EXEC_ENGINE_PINNED))
2697 return;
2698
2699 eb->args->flags &= ~__EXEC_ENGINE_PINNED;
2700
2701 mutex_lock(&tl->mutex);
2702 intel_context_exit(ce);
2703 mutex_unlock(&tl->mutex);
2704
2705 intel_context_unpin(ce);
2706}
2707
2708static unsigned int
2709eb_select_legacy_ring(struct i915_execbuffer *eb)
2710{
2711 struct drm_i915_private *i915 = eb->i915;
2712 struct drm_i915_gem_execbuffer2 *args = eb->args;
2713 unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
2714
2715 if (user_ring_id != I915_EXEC_BSD &&
2716 (args->flags & I915_EXEC_BSD_MASK)) {
2717 drm_dbg(&i915->drm,
2718 "execbuf with non bsd ring but with invalid "
2719 "bsd dispatch flags: %d\n", (int)(args->flags));
2720 return -1;
2721 }
2722
2723 if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) {
2724 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
2725
2726 if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
2727 bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file);
2728 } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
2729 bsd_idx <= I915_EXEC_BSD_RING2) {
2730 bsd_idx >>= I915_EXEC_BSD_SHIFT;
2731 bsd_idx--;
2732 } else {
2733 drm_dbg(&i915->drm,
2734 "execbuf with unknown bsd ring: %u\n",
2735 bsd_idx);
2736 return -1;
2737 }
2738
2739 return _VCS(bsd_idx);
2740 }
2741
2742 if (user_ring_id >= ARRAY_SIZE(user_ring_map)) {
2743 drm_dbg(&i915->drm, "execbuf with unknown ring: %u\n",
2744 user_ring_id);
2745 return -1;
2746 }
2747
2748 return user_ring_map[user_ring_id];
2749}
2750
2751static int
2752eb_select_engine(struct i915_execbuffer *eb)
2753{
2754 struct intel_context *ce;
2755 unsigned int idx;
2756 int err;
2757
2758 if (i915_gem_context_user_engines(eb->gem_context))
2759 idx = eb->args->flags & I915_EXEC_RING_MASK;
2760 else
2761 idx = eb_select_legacy_ring(eb);
2762
2763 ce = i915_gem_context_get_engine(eb->gem_context, idx);
2764 if (IS_ERR(ce))
2765 return PTR_ERR(ce);
2766
2767 intel_gt_pm_get(ce->engine->gt);
2768
2769 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
2770 err = intel_context_alloc_state(ce);
2771 if (err)
2772 goto err;
2773 }
2774
2775
2776
2777
2778
2779 err = intel_gt_terminally_wedged(ce->engine->gt);
2780 if (err)
2781 goto err;
2782
2783 eb->context = ce;
2784 eb->engine = ce->engine;
2785
2786
2787
2788
2789
2790
2791 return err;
2792
2793err:
2794 intel_gt_pm_put(ce->engine->gt);
2795 intel_context_put(ce);
2796 return err;
2797}
2798
2799static void
2800eb_put_engine(struct i915_execbuffer *eb)
2801{
2802 intel_gt_pm_put(eb->engine->gt);
2803 intel_context_put(eb->context);
2804}
2805
2806static void
2807__free_fence_array(struct eb_fence *fences, unsigned int n)
2808{
2809 while (n--) {
2810 drm_syncobj_put(ptr_mask_bits(fences[n].syncobj, 2));
2811 dma_fence_put(fences[n].dma_fence);
2812 kfree(fences[n].chain_fence);
2813 }
2814 kvfree(fences);
2815}
2816
2817static int
2818add_timeline_fence_array(struct i915_execbuffer *eb,
2819 const struct drm_i915_gem_execbuffer_ext_timeline_fences *timeline_fences)
2820{
2821 struct drm_i915_gem_exec_fence __user *user_fences;
2822 u64 __user *user_values;
2823 struct eb_fence *f;
2824 u64 nfences;
2825 int err = 0;
2826
2827 nfences = timeline_fences->fence_count;
2828 if (!nfences)
2829 return 0;
2830
2831
2832 BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2833 if (nfences > min_t(unsigned long,
2834 ULONG_MAX / sizeof(*user_fences),
2835 SIZE_MAX / sizeof(*f)) - eb->num_fences)
2836 return -EINVAL;
2837
2838 user_fences = u64_to_user_ptr(timeline_fences->handles_ptr);
2839 if (!access_ok(user_fences, nfences * sizeof(*user_fences)))
2840 return -EFAULT;
2841
2842 user_values = u64_to_user_ptr(timeline_fences->values_ptr);
2843 if (!access_ok(user_values, nfences * sizeof(*user_values)))
2844 return -EFAULT;
2845
2846 f = krealloc(eb->fences,
2847 (eb->num_fences + nfences) * sizeof(*f),
2848 __GFP_NOWARN | GFP_KERNEL);
2849 if (!f)
2850 return -ENOMEM;
2851
2852 eb->fences = f;
2853 f += eb->num_fences;
2854
2855 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
2856 ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
2857
2858 while (nfences--) {
2859 struct drm_i915_gem_exec_fence user_fence;
2860 struct drm_syncobj *syncobj;
2861 struct dma_fence *fence = NULL;
2862 u64 point;
2863
2864 if (__copy_from_user(&user_fence,
2865 user_fences++,
2866 sizeof(user_fence)))
2867 return -EFAULT;
2868
2869 if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
2870 return -EINVAL;
2871
2872 if (__get_user(point, user_values++))
2873 return -EFAULT;
2874
2875 syncobj = drm_syncobj_find(eb->file, user_fence.handle);
2876 if (!syncobj) {
2877 DRM_DEBUG("Invalid syncobj handle provided\n");
2878 return -ENOENT;
2879 }
2880
2881 fence = drm_syncobj_fence_get(syncobj);
2882
2883 if (!fence && user_fence.flags &&
2884 !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2885 DRM_DEBUG("Syncobj handle has no fence\n");
2886 drm_syncobj_put(syncobj);
2887 return -EINVAL;
2888 }
2889
2890 if (fence)
2891 err = dma_fence_chain_find_seqno(&fence, point);
2892
2893 if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2894 DRM_DEBUG("Syncobj handle missing requested point %llu\n", point);
2895 dma_fence_put(fence);
2896 drm_syncobj_put(syncobj);
2897 return err;
2898 }
2899
2900
2901
2902
2903
2904
2905 if (!fence && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
2906 drm_syncobj_put(syncobj);
2907 continue;
2908 }
2909
2910
2911
2912
2913
2914 if (point != 0 && user_fence.flags & I915_EXEC_FENCE_SIGNAL) {
2915
2916
2917
2918
2919 if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
2920 DRM_DEBUG("Trying to wait & signal the same timeline point.\n");
2921 dma_fence_put(fence);
2922 drm_syncobj_put(syncobj);
2923 return -EINVAL;
2924 }
2925
2926 f->chain_fence =
2927 kmalloc(sizeof(*f->chain_fence),
2928 GFP_KERNEL);
2929 if (!f->chain_fence) {
2930 drm_syncobj_put(syncobj);
2931 dma_fence_put(fence);
2932 return -ENOMEM;
2933 }
2934 } else {
2935 f->chain_fence = NULL;
2936 }
2937
2938 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
2939 f->dma_fence = fence;
2940 f->value = point;
2941 f++;
2942 eb->num_fences++;
2943 }
2944
2945 return 0;
2946}
2947
2948static int add_fence_array(struct i915_execbuffer *eb)
2949{
2950 struct drm_i915_gem_execbuffer2 *args = eb->args;
2951 struct drm_i915_gem_exec_fence __user *user;
2952 unsigned long num_fences = args->num_cliprects;
2953 struct eb_fence *f;
2954
2955 if (!(args->flags & I915_EXEC_FENCE_ARRAY))
2956 return 0;
2957
2958 if (!num_fences)
2959 return 0;
2960
2961
2962 BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2963 if (num_fences > min_t(unsigned long,
2964 ULONG_MAX / sizeof(*user),
2965 SIZE_MAX / sizeof(*f) - eb->num_fences))
2966 return -EINVAL;
2967
2968 user = u64_to_user_ptr(args->cliprects_ptr);
2969 if (!access_ok(user, num_fences * sizeof(*user)))
2970 return -EFAULT;
2971
2972 f = krealloc(eb->fences,
2973 (eb->num_fences + num_fences) * sizeof(*f),
2974 __GFP_NOWARN | GFP_KERNEL);
2975 if (!f)
2976 return -ENOMEM;
2977
2978 eb->fences = f;
2979 f += eb->num_fences;
2980 while (num_fences--) {
2981 struct drm_i915_gem_exec_fence user_fence;
2982 struct drm_syncobj *syncobj;
2983 struct dma_fence *fence = NULL;
2984
2985 if (__copy_from_user(&user_fence, user++, sizeof(user_fence)))
2986 return -EFAULT;
2987
2988 if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
2989 return -EINVAL;
2990
2991 syncobj = drm_syncobj_find(eb->file, user_fence.handle);
2992 if (!syncobj) {
2993 DRM_DEBUG("Invalid syncobj handle provided\n");
2994 return -ENOENT;
2995 }
2996
2997 if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
2998 fence = drm_syncobj_fence_get(syncobj);
2999 if (!fence) {
3000 DRM_DEBUG("Syncobj handle has no fence\n");
3001 drm_syncobj_put(syncobj);
3002 return -EINVAL;
3003 }
3004 }
3005
3006 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
3007 ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
3008
3009 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
3010 f->dma_fence = fence;
3011 f->value = 0;
3012 f->chain_fence = NULL;
3013 f++;
3014 eb->num_fences++;
3015 }
3016
3017 return 0;
3018}
3019
3020static void put_fence_array(struct eb_fence *fences, int num_fences)
3021{
3022 if (fences)
3023 __free_fence_array(fences, num_fences);
3024}
3025
3026static int
3027await_fence_array(struct i915_execbuffer *eb)
3028{
3029 unsigned int n;
3030 int err;
3031
3032 for (n = 0; n < eb->num_fences; n++) {
3033 struct drm_syncobj *syncobj;
3034 unsigned int flags;
3035
3036 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2);
3037
3038 if (!eb->fences[n].dma_fence)
3039 continue;
3040
3041 err = i915_request_await_dma_fence(eb->request,
3042 eb->fences[n].dma_fence);
3043 if (err < 0)
3044 return err;
3045 }
3046
3047 return 0;
3048}
3049
3050static void signal_fence_array(const struct i915_execbuffer *eb)
3051{
3052 struct dma_fence * const fence = &eb->request->fence;
3053 unsigned int n;
3054
3055 for (n = 0; n < eb->num_fences; n++) {
3056 struct drm_syncobj *syncobj;
3057 unsigned int flags;
3058
3059 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2);
3060 if (!(flags & I915_EXEC_FENCE_SIGNAL))
3061 continue;
3062
3063 if (eb->fences[n].chain_fence) {
3064 drm_syncobj_add_point(syncobj,
3065 eb->fences[n].chain_fence,
3066 fence,
3067 eb->fences[n].value);
3068
3069
3070
3071
3072 eb->fences[n].chain_fence = NULL;
3073 } else {
3074 drm_syncobj_replace_fence(syncobj, fence);
3075 }
3076 }
3077}
3078
3079static int
3080parse_timeline_fences(struct i915_user_extension __user *ext, void *data)
3081{
3082 struct i915_execbuffer *eb = data;
3083 struct drm_i915_gem_execbuffer_ext_timeline_fences timeline_fences;
3084
3085 if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences)))
3086 return -EFAULT;
3087
3088 return add_timeline_fence_array(eb, &timeline_fences);
3089}
3090
3091static void retire_requests(struct intel_timeline *tl, struct i915_request *end)
3092{
3093 struct i915_request *rq, *rn;
3094
3095 list_for_each_entry_safe(rq, rn, &tl->requests, link)
3096 if (rq == end || !i915_request_retire(rq))
3097 break;
3098}
3099
3100static int eb_request_add(struct i915_execbuffer *eb, int err)
3101{
3102 struct i915_request *rq = eb->request;
3103 struct intel_timeline * const tl = i915_request_timeline(rq);
3104 struct i915_sched_attr attr = {};
3105 struct i915_request *prev;
3106
3107 lockdep_assert_held(&tl->mutex);
3108 lockdep_unpin_lock(&tl->mutex, rq->cookie);
3109
3110 trace_i915_request_add(rq);
3111
3112 prev = __i915_request_commit(rq);
3113
3114
3115 if (likely(!intel_context_is_closed(eb->context))) {
3116 attr = eb->gem_context->sched;
3117 } else {
3118
3119 i915_request_set_error_once(rq, -ENOENT);
3120 __i915_request_skip(rq);
3121 err = -ENOENT;
3122 }
3123
3124 __i915_request_queue(rq, &attr);
3125
3126
3127 if (prev)
3128 retire_requests(tl, prev);
3129
3130 mutex_unlock(&tl->mutex);
3131
3132 return err;
3133}
3134
3135static const i915_user_extension_fn execbuf_extensions[] = {
3136 [DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES] = parse_timeline_fences,
3137};
3138
3139static int
3140parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args,
3141 struct i915_execbuffer *eb)
3142{
3143 if (!(args->flags & I915_EXEC_USE_EXTENSIONS))
3144 return 0;
3145
3146
3147
3148
3149 if (eb->args->flags & I915_EXEC_FENCE_ARRAY)
3150 return -EINVAL;
3151
3152 if (args->num_cliprects != 0)
3153 return -EINVAL;
3154
3155 return i915_user_extensions(u64_to_user_ptr(args->cliprects_ptr),
3156 execbuf_extensions,
3157 ARRAY_SIZE(execbuf_extensions),
3158 eb);
3159}
3160
3161static int
3162i915_gem_do_execbuffer(struct drm_device *dev,
3163 struct drm_file *file,
3164 struct drm_i915_gem_execbuffer2 *args,
3165 struct drm_i915_gem_exec_object2 *exec)
3166{
3167 struct drm_i915_private *i915 = to_i915(dev);
3168 struct i915_execbuffer eb;
3169 struct dma_fence *in_fence = NULL;
3170 struct sync_file *out_fence = NULL;
3171 struct i915_vma *batch;
3172 int out_fence_fd = -1;
3173 int err;
3174
3175 BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS);
3176 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
3177 ~__EXEC_OBJECT_UNKNOWN_FLAGS);
3178
3179 eb.i915 = i915;
3180 eb.file = file;
3181 eb.args = args;
3182 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
3183 args->flags |= __EXEC_HAS_RELOC;
3184
3185 eb.exec = exec;
3186 eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1);
3187 eb.vma[0].vma = NULL;
3188 eb.reloc_pool = eb.batch_pool = NULL;
3189 eb.reloc_context = NULL;
3190
3191 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
3192 reloc_cache_init(&eb.reloc_cache, eb.i915);
3193
3194 eb.buffer_count = args->buffer_count;
3195 eb.batch_start_offset = args->batch_start_offset;
3196 eb.batch_len = args->batch_len;
3197 eb.trampoline = NULL;
3198
3199 eb.fences = NULL;
3200 eb.num_fences = 0;
3201
3202 eb.batch_flags = 0;
3203 if (args->flags & I915_EXEC_SECURE) {
3204 if (INTEL_GEN(i915) >= 11)
3205 return -ENODEV;
3206
3207
3208 if (!HAS_SECURE_BATCHES(i915))
3209 return -EPERM;
3210
3211 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
3212 return -EPERM;
3213
3214 eb.batch_flags |= I915_DISPATCH_SECURE;
3215 }
3216 if (args->flags & I915_EXEC_IS_PINNED)
3217 eb.batch_flags |= I915_DISPATCH_PINNED;
3218
3219 err = parse_execbuf2_extensions(args, &eb);
3220 if (err)
3221 goto err_ext;
3222
3223 err = add_fence_array(&eb);
3224 if (err)
3225 goto err_ext;
3226
3227#define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT)
3228 if (args->flags & IN_FENCES) {
3229 if ((args->flags & IN_FENCES) == IN_FENCES)
3230 return -EINVAL;
3231
3232 in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
3233 if (!in_fence) {
3234 err = -EINVAL;
3235 goto err_ext;
3236 }
3237 }
3238#undef IN_FENCES
3239
3240 if (args->flags & I915_EXEC_FENCE_OUT) {
3241 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
3242 if (out_fence_fd < 0) {
3243 err = out_fence_fd;
3244 goto err_in_fence;
3245 }
3246 }
3247
3248 err = eb_create(&eb);
3249 if (err)
3250 goto err_out_fence;
3251
3252 GEM_BUG_ON(!eb.lut_size);
3253
3254 err = eb_select_context(&eb);
3255 if (unlikely(err))
3256 goto err_destroy;
3257
3258 err = eb_select_engine(&eb);
3259 if (unlikely(err))
3260 goto err_context;
3261
3262 err = eb_lookup_vmas(&eb);
3263 if (err) {
3264 eb_release_vmas(&eb, true);
3265 goto err_engine;
3266 }
3267
3268 i915_gem_ww_ctx_init(&eb.ww, true);
3269
3270 err = eb_relocate_parse(&eb);
3271 if (err) {
3272
3273
3274
3275
3276
3277
3278
3279 args->flags &= ~__EXEC_HAS_RELOC;
3280 goto err_vma;
3281 }
3282
3283 ww_acquire_done(&eb.ww.ctx);
3284
3285 batch = eb.batch->vma;
3286
3287
3288 GEM_BUG_ON(eb.reloc_cache.rq);
3289
3290
3291 eb.request = i915_request_create(eb.context);
3292 if (IS_ERR(eb.request)) {
3293 err = PTR_ERR(eb.request);
3294 goto err_vma;
3295 }
3296
3297 if (in_fence) {
3298 if (args->flags & I915_EXEC_FENCE_SUBMIT)
3299 err = i915_request_await_execution(eb.request,
3300 in_fence,
3301 eb.engine->bond_execute);
3302 else
3303 err = i915_request_await_dma_fence(eb.request,
3304 in_fence);
3305 if (err < 0)
3306 goto err_request;
3307 }
3308
3309 if (eb.fences) {
3310 err = await_fence_array(&eb);
3311 if (err)
3312 goto err_request;
3313 }
3314
3315 if (out_fence_fd != -1) {
3316 out_fence = sync_file_create(&eb.request->fence);
3317 if (!out_fence) {
3318 err = -ENOMEM;
3319 goto err_request;
3320 }
3321 }
3322
3323
3324
3325
3326
3327
3328
3329
3330 eb.request->batch = batch;
3331 if (eb.batch_pool)
3332 intel_gt_buffer_pool_mark_active(eb.batch_pool, eb.request);
3333
3334 trace_i915_request_queue(eb.request, eb.batch_flags);
3335 err = eb_submit(&eb, batch);
3336err_request:
3337 i915_request_get(eb.request);
3338 err = eb_request_add(&eb, err);
3339
3340 if (eb.fences)
3341 signal_fence_array(&eb);
3342
3343 if (out_fence) {
3344 if (err == 0) {
3345 fd_install(out_fence_fd, out_fence->file);
3346 args->rsvd2 &= GENMASK_ULL(31, 0);
3347 args->rsvd2 |= (u64)out_fence_fd << 32;
3348 out_fence_fd = -1;
3349 } else {
3350 fput(out_fence->file);
3351 }
3352 }
3353 i915_request_put(eb.request);
3354
3355err_vma:
3356 eb_release_vmas(&eb, true);
3357 if (eb.trampoline)
3358 i915_vma_unpin(eb.trampoline);
3359 WARN_ON(err == -EDEADLK);
3360 i915_gem_ww_ctx_fini(&eb.ww);
3361
3362 if (eb.batch_pool)
3363 intel_gt_buffer_pool_put(eb.batch_pool);
3364 if (eb.reloc_pool)
3365 intel_gt_buffer_pool_put(eb.reloc_pool);
3366 if (eb.reloc_context)
3367 intel_context_put(eb.reloc_context);
3368err_engine:
3369 eb_put_engine(&eb);
3370err_context:
3371 i915_gem_context_put(eb.gem_context);
3372err_destroy:
3373 eb_destroy(&eb);
3374err_out_fence:
3375 if (out_fence_fd != -1)
3376 put_unused_fd(out_fence_fd);
3377err_in_fence:
3378 dma_fence_put(in_fence);
3379err_ext:
3380 put_fence_array(eb.fences, eb.num_fences);
3381 return err;
3382}
3383
3384static size_t eb_element_size(void)
3385{
3386 return sizeof(struct drm_i915_gem_exec_object2) + sizeof(struct eb_vma);
3387}
3388
3389static bool check_buffer_count(size_t count)
3390{
3391 const size_t sz = eb_element_size();
3392
3393
3394
3395
3396
3397
3398
3399 return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
3400}
3401
3402
3403
3404
3405
3406int
3407i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
3408 struct drm_file *file)
3409{
3410 struct drm_i915_private *i915 = to_i915(dev);
3411 struct drm_i915_gem_execbuffer *args = data;
3412 struct drm_i915_gem_execbuffer2 exec2;
3413 struct drm_i915_gem_exec_object *exec_list = NULL;
3414 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3415 const size_t count = args->buffer_count;
3416 unsigned int i;
3417 int err;
3418
3419 if (!check_buffer_count(count)) {
3420 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count);
3421 return -EINVAL;
3422 }
3423
3424 exec2.buffers_ptr = args->buffers_ptr;
3425 exec2.buffer_count = args->buffer_count;
3426 exec2.batch_start_offset = args->batch_start_offset;
3427 exec2.batch_len = args->batch_len;
3428 exec2.DR1 = args->DR1;
3429 exec2.DR4 = args->DR4;
3430 exec2.num_cliprects = args->num_cliprects;
3431 exec2.cliprects_ptr = args->cliprects_ptr;
3432 exec2.flags = I915_EXEC_RENDER;
3433 i915_execbuffer2_set_context_id(exec2, 0);
3434
3435 err = i915_gem_check_execbuffer(&exec2);
3436 if (err)
3437 return err;
3438
3439
3440 exec_list = kvmalloc_array(count, sizeof(*exec_list),
3441 __GFP_NOWARN | GFP_KERNEL);
3442
3443
3444 exec2_list = kvmalloc_array(count + 2, eb_element_size(),
3445 __GFP_NOWARN | GFP_KERNEL);
3446 if (exec_list == NULL || exec2_list == NULL) {
3447 drm_dbg(&i915->drm,
3448 "Failed to allocate exec list for %d buffers\n",
3449 args->buffer_count);
3450 kvfree(exec_list);
3451 kvfree(exec2_list);
3452 return -ENOMEM;
3453 }
3454 err = copy_from_user(exec_list,
3455 u64_to_user_ptr(args->buffers_ptr),
3456 sizeof(*exec_list) * count);
3457 if (err) {
3458 drm_dbg(&i915->drm, "copy %d exec entries failed %d\n",
3459 args->buffer_count, err);
3460 kvfree(exec_list);
3461 kvfree(exec2_list);
3462 return -EFAULT;
3463 }
3464
3465 for (i = 0; i < args->buffer_count; i++) {
3466 exec2_list[i].handle = exec_list[i].handle;
3467 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3468 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3469 exec2_list[i].alignment = exec_list[i].alignment;
3470 exec2_list[i].offset = exec_list[i].offset;
3471 if (INTEL_GEN(to_i915(dev)) < 4)
3472 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3473 else
3474 exec2_list[i].flags = 0;
3475 }
3476
3477 err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
3478 if (exec2.flags & __EXEC_HAS_RELOC) {
3479 struct drm_i915_gem_exec_object __user *user_exec_list =
3480 u64_to_user_ptr(args->buffers_ptr);
3481
3482
3483 for (i = 0; i < args->buffer_count; i++) {
3484 if (!(exec2_list[i].offset & UPDATE))
3485 continue;
3486
3487 exec2_list[i].offset =
3488 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
3489 exec2_list[i].offset &= PIN_OFFSET_MASK;
3490 if (__copy_to_user(&user_exec_list[i].offset,
3491 &exec2_list[i].offset,
3492 sizeof(user_exec_list[i].offset)))
3493 break;
3494 }
3495 }
3496
3497 kvfree(exec_list);
3498 kvfree(exec2_list);
3499 return err;
3500}
3501
3502int
3503i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
3504 struct drm_file *file)
3505{
3506 struct drm_i915_private *i915 = to_i915(dev);
3507 struct drm_i915_gem_execbuffer2 *args = data;
3508 struct drm_i915_gem_exec_object2 *exec2_list;
3509 const size_t count = args->buffer_count;
3510 int err;
3511
3512 if (!check_buffer_count(count)) {
3513 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count);
3514 return -EINVAL;
3515 }
3516
3517 err = i915_gem_check_execbuffer(args);
3518 if (err)
3519 return err;
3520
3521
3522 exec2_list = kvmalloc_array(count + 2, eb_element_size(),
3523 __GFP_NOWARN | GFP_KERNEL);
3524 if (exec2_list == NULL) {
3525 drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n",
3526 count);
3527 return -ENOMEM;
3528 }
3529 if (copy_from_user(exec2_list,
3530 u64_to_user_ptr(args->buffers_ptr),
3531 sizeof(*exec2_list) * count)) {
3532 drm_dbg(&i915->drm, "copy %zd exec entries failed\n", count);
3533 kvfree(exec2_list);
3534 return -EFAULT;
3535 }
3536
3537 err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
3538
3539
3540
3541
3542
3543
3544
3545 if (args->flags & __EXEC_HAS_RELOC) {
3546 struct drm_i915_gem_exec_object2 __user *user_exec_list =
3547 u64_to_user_ptr(args->buffers_ptr);
3548 unsigned int i;
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558 if (!user_write_access_begin(user_exec_list,
3559 count * sizeof(*user_exec_list)))
3560 goto end;
3561
3562 for (i = 0; i < args->buffer_count; i++) {
3563 if (!(exec2_list[i].offset & UPDATE))
3564 continue;
3565
3566 exec2_list[i].offset =
3567 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
3568 unsafe_put_user(exec2_list[i].offset,
3569 &user_exec_list[i].offset,
3570 end_user);
3571 }
3572end_user:
3573 user_write_access_end();
3574end:;
3575 }
3576
3577 args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
3578 kvfree(exec2_list);
3579 return err;
3580}
3581
3582#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
3583#include "selftests/i915_gem_execbuffer.c"
3584#endif
3585