1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include "drmP.h"
30#include "drm.h"
31#include "i915_drm.h"
32#include "i915_drv.h"
33#include "i915_trace.h"
34#include "intel_drv.h"
35
36struct change_domains {
37 uint32_t invalidate_domains;
38 uint32_t flush_domains;
39 uint32_t flush_rings;
40};
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153static void
154i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
155 struct intel_ring_buffer *ring,
156 struct change_domains *cd)
157{
158 uint32_t invalidate_domains = 0, flush_domains = 0;
159
160
161
162
163
164 if (obj->base.pending_write_domain == 0)
165 obj->base.pending_read_domains |= obj->base.read_domains;
166
167
168
169
170
171
172
173 if (obj->base.write_domain &&
174 (((obj->base.write_domain != obj->base.pending_read_domains ||
175 obj->ring != ring)) ||
176 (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
177 flush_domains |= obj->base.write_domain;
178 invalidate_domains |=
179 obj->base.pending_read_domains & ~obj->base.write_domain;
180 }
181
182
183
184
185 invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
186 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
187 i915_gem_clflush_object(obj);
188
189
190 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
191 i915_gem_release_mmap(obj);
192
193
194
195
196
197
198
199 if (flush_domains == 0 && obj->base.pending_write_domain == 0)
200 obj->base.pending_write_domain = obj->base.write_domain;
201
202 cd->invalidate_domains |= invalidate_domains;
203 cd->flush_domains |= flush_domains;
204 if (flush_domains & I915_GEM_GPU_DOMAINS)
205 cd->flush_rings |= obj->ring->id;
206 if (invalidate_domains & I915_GEM_GPU_DOMAINS)
207 cd->flush_rings |= ring->id;
208}
209
210struct eb_objects {
211 int and;
212 struct hlist_head buckets[0];
213};
214
215static struct eb_objects *
216eb_create(int size)
217{
218 struct eb_objects *eb;
219 int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
220 while (count > size)
221 count >>= 1;
222 eb = kzalloc(count*sizeof(struct hlist_head) +
223 sizeof(struct eb_objects),
224 GFP_KERNEL);
225 if (eb == NULL)
226 return eb;
227
228 eb->and = count - 1;
229 return eb;
230}
231
232static void
233eb_reset(struct eb_objects *eb)
234{
235 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
236}
237
238static void
239eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj)
240{
241 hlist_add_head(&obj->exec_node,
242 &eb->buckets[obj->exec_handle & eb->and]);
243}
244
245static struct drm_i915_gem_object *
246eb_get_object(struct eb_objects *eb, unsigned long handle)
247{
248 struct hlist_head *head;
249 struct hlist_node *node;
250 struct drm_i915_gem_object *obj;
251
252 head = &eb->buckets[handle & eb->and];
253 hlist_for_each(node, head) {
254 obj = hlist_entry(node, struct drm_i915_gem_object, exec_node);
255 if (obj->exec_handle == handle)
256 return obj;
257 }
258
259 return NULL;
260}
261
262static void
263eb_destroy(struct eb_objects *eb)
264{
265 kfree(eb);
266}
267
268static int
269i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
270 struct eb_objects *eb,
271 struct drm_i915_gem_relocation_entry *reloc)
272{
273 struct drm_device *dev = obj->base.dev;
274 struct drm_gem_object *target_obj;
275 uint32_t target_offset;
276 int ret = -EINVAL;
277
278
279 target_obj = &eb_get_object(eb, reloc->target_handle)->base;
280 if (unlikely(target_obj == NULL))
281 return -ENOENT;
282
283 target_offset = to_intel_bo(target_obj)->gtt_offset;
284
285#if WATCH_RELOC
286 DRM_INFO("%s: obj %p offset %08x target %d "
287 "read %08x write %08x gtt %08x "
288 "presumed %08x delta %08x\n",
289 __func__,
290 obj,
291 (int) reloc->offset,
292 (int) reloc->target_handle,
293 (int) reloc->read_domains,
294 (int) reloc->write_domain,
295 (int) target_offset,
296 (int) reloc->presumed_offset,
297 reloc->delta);
298#endif
299
300
301
302
303 if (unlikely(target_offset == 0)) {
304 DRM_ERROR("No GTT space found for object %d\n",
305 reloc->target_handle);
306 return ret;
307 }
308
309
310 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
311 DRM_ERROR("reloc with multiple write domains: "
312 "obj %p target %d offset %d "
313 "read %08x write %08x",
314 obj, reloc->target_handle,
315 (int) reloc->offset,
316 reloc->read_domains,
317 reloc->write_domain);
318 return ret;
319 }
320 if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) {
321 DRM_ERROR("reloc with read/write CPU domains: "
322 "obj %p target %d offset %d "
323 "read %08x write %08x",
324 obj, reloc->target_handle,
325 (int) reloc->offset,
326 reloc->read_domains,
327 reloc->write_domain);
328 return ret;
329 }
330 if (unlikely(reloc->write_domain && target_obj->pending_write_domain &&
331 reloc->write_domain != target_obj->pending_write_domain)) {
332 DRM_ERROR("Write domain conflict: "
333 "obj %p target %d offset %d "
334 "new %08x old %08x\n",
335 obj, reloc->target_handle,
336 (int) reloc->offset,
337 reloc->write_domain,
338 target_obj->pending_write_domain);
339 return ret;
340 }
341
342 target_obj->pending_read_domains |= reloc->read_domains;
343 target_obj->pending_write_domain |= reloc->write_domain;
344
345
346
347
348 if (target_offset == reloc->presumed_offset)
349 return 0;
350
351
352 if (unlikely(reloc->offset > obj->base.size - 4)) {
353 DRM_ERROR("Relocation beyond object bounds: "
354 "obj %p target %d offset %d size %d.\n",
355 obj, reloc->target_handle,
356 (int) reloc->offset,
357 (int) obj->base.size);
358 return ret;
359 }
360 if (unlikely(reloc->offset & 3)) {
361 DRM_ERROR("Relocation not 4-byte aligned: "
362 "obj %p target %d offset %d.\n",
363 obj, reloc->target_handle,
364 (int) reloc->offset);
365 return ret;
366 }
367
368
369 if (unlikely(reloc->delta >= target_obj->size)) {
370 DRM_ERROR("Relocation beyond target object bounds: "
371 "obj %p target %d delta %d size %d.\n",
372 obj, reloc->target_handle,
373 (int) reloc->delta,
374 (int) target_obj->size);
375 return ret;
376 }
377
378 reloc->delta += target_offset;
379 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
380 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
381 char *vaddr;
382
383 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
384 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
385 kunmap_atomic(vaddr);
386 } else {
387 struct drm_i915_private *dev_priv = dev->dev_private;
388 uint32_t __iomem *reloc_entry;
389 void __iomem *reloc_page;
390
391 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
392 if (ret)
393 return ret;
394
395
396 reloc->offset += obj->gtt_offset;
397 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
398 reloc->offset & PAGE_MASK);
399 reloc_entry = (uint32_t __iomem *)
400 (reloc_page + (reloc->offset & ~PAGE_MASK));
401 iowrite32(reloc->delta, reloc_entry);
402 io_mapping_unmap_atomic(reloc_page);
403 }
404
405
406 reloc->presumed_offset = target_offset;
407
408 return 0;
409}
410
411static int
412i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
413 struct eb_objects *eb)
414{
415 struct drm_i915_gem_relocation_entry __user *user_relocs;
416 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
417 int i, ret;
418
419 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
420 for (i = 0; i < entry->relocation_count; i++) {
421 struct drm_i915_gem_relocation_entry reloc;
422
423 if (__copy_from_user_inatomic(&reloc,
424 user_relocs+i,
425 sizeof(reloc)))
426 return -EFAULT;
427
428 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc);
429 if (ret)
430 return ret;
431
432 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
433 &reloc.presumed_offset,
434 sizeof(reloc.presumed_offset)))
435 return -EFAULT;
436 }
437
438 return 0;
439}
440
441static int
442i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
443 struct eb_objects *eb,
444 struct drm_i915_gem_relocation_entry *relocs)
445{
446 const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
447 int i, ret;
448
449 for (i = 0; i < entry->relocation_count; i++) {
450 ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]);
451 if (ret)
452 return ret;
453 }
454
455 return 0;
456}
457
458static int
459i915_gem_execbuffer_relocate(struct drm_device *dev,
460 struct eb_objects *eb,
461 struct list_head *objects)
462{
463 struct drm_i915_gem_object *obj;
464 int ret;
465
466 list_for_each_entry(obj, objects, exec_list) {
467 ret = i915_gem_execbuffer_relocate_object(obj, eb);
468 if (ret)
469 return ret;
470 }
471
472 return 0;
473}
474
475static int
476i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
477 struct drm_file *file,
478 struct list_head *objects)
479{
480 struct drm_i915_gem_object *obj;
481 int ret, retry;
482 bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
483 struct list_head ordered_objects;
484
485 INIT_LIST_HEAD(&ordered_objects);
486 while (!list_empty(objects)) {
487 struct drm_i915_gem_exec_object2 *entry;
488 bool need_fence, need_mappable;
489
490 obj = list_first_entry(objects,
491 struct drm_i915_gem_object,
492 exec_list);
493 entry = obj->exec_entry;
494
495 need_fence =
496 has_fenced_gpu_access &&
497 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
498 obj->tiling_mode != I915_TILING_NONE;
499 need_mappable =
500 entry->relocation_count ? true : need_fence;
501
502 if (need_mappable)
503 list_move(&obj->exec_list, &ordered_objects);
504 else
505 list_move_tail(&obj->exec_list, &ordered_objects);
506
507 obj->base.pending_read_domains = 0;
508 obj->base.pending_write_domain = 0;
509 }
510 list_splice(&ordered_objects, objects);
511
512
513
514
515
516
517
518
519
520
521
522
523
524 retry = 0;
525 do {
526 ret = 0;
527
528
529 list_for_each_entry(obj, objects, exec_list) {
530 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
531 bool need_fence, need_mappable;
532 if (!obj->gtt_space)
533 continue;
534
535 need_fence =
536 has_fenced_gpu_access &&
537 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
538 obj->tiling_mode != I915_TILING_NONE;
539 need_mappable =
540 entry->relocation_count ? true : need_fence;
541
542 if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
543 (need_mappable && !obj->map_and_fenceable))
544 ret = i915_gem_object_unbind(obj);
545 else
546 ret = i915_gem_object_pin(obj,
547 entry->alignment,
548 need_mappable);
549 if (ret)
550 goto err;
551
552 entry++;
553 }
554
555
556 list_for_each_entry(obj, objects, exec_list) {
557 struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
558 bool need_fence;
559
560 need_fence =
561 has_fenced_gpu_access &&
562 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
563 obj->tiling_mode != I915_TILING_NONE;
564
565 if (!obj->gtt_space) {
566 bool need_mappable =
567 entry->relocation_count ? true : need_fence;
568
569 ret = i915_gem_object_pin(obj,
570 entry->alignment,
571 need_mappable);
572 if (ret)
573 break;
574 }
575
576 if (has_fenced_gpu_access) {
577 if (need_fence) {
578 ret = i915_gem_object_get_fence(obj, ring, 1);
579 if (ret)
580 break;
581 } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
582 obj->tiling_mode == I915_TILING_NONE) {
583
584 ret = i915_gem_object_put_fence(obj);
585 if (ret)
586 break;
587 }
588 obj->pending_fenced_gpu_access = need_fence;
589 }
590
591 entry->offset = obj->gtt_offset;
592 }
593
594
595 list_for_each_entry(obj, objects, exec_list) {
596 if (obj->gtt_space)
597 i915_gem_object_unpin(obj);
598 }
599
600 if (ret != -ENOSPC || retry > 1)
601 return ret;
602
603
604
605
606 ret = i915_gem_evict_everything(ring->dev, retry == 0);
607 if (ret)
608 return ret;
609
610 retry++;
611 } while (1);
612
613err:
614 obj = list_entry(obj->exec_list.prev,
615 struct drm_i915_gem_object,
616 exec_list);
617 while (objects != &obj->exec_list) {
618 if (obj->gtt_space)
619 i915_gem_object_unpin(obj);
620
621 obj = list_entry(obj->exec_list.prev,
622 struct drm_i915_gem_object,
623 exec_list);
624 }
625
626 return ret;
627}
628
629static int
630i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
631 struct drm_file *file,
632 struct intel_ring_buffer *ring,
633 struct list_head *objects,
634 struct eb_objects *eb,
635 struct drm_i915_gem_exec_object2 *exec,
636 int count)
637{
638 struct drm_i915_gem_relocation_entry *reloc;
639 struct drm_i915_gem_object *obj;
640 int *reloc_offset;
641 int i, total, ret;
642
643
644 while (!list_empty(objects)) {
645 obj = list_first_entry(objects,
646 struct drm_i915_gem_object,
647 exec_list);
648 list_del_init(&obj->exec_list);
649 drm_gem_object_unreference(&obj->base);
650 }
651
652 mutex_unlock(&dev->struct_mutex);
653
654 total = 0;
655 for (i = 0; i < count; i++)
656 total += exec[i].relocation_count;
657
658 reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
659 reloc = drm_malloc_ab(total, sizeof(*reloc));
660 if (reloc == NULL || reloc_offset == NULL) {
661 drm_free_large(reloc);
662 drm_free_large(reloc_offset);
663 mutex_lock(&dev->struct_mutex);
664 return -ENOMEM;
665 }
666
667 total = 0;
668 for (i = 0; i < count; i++) {
669 struct drm_i915_gem_relocation_entry __user *user_relocs;
670
671 user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr;
672
673 if (copy_from_user(reloc+total, user_relocs,
674 exec[i].relocation_count * sizeof(*reloc))) {
675 ret = -EFAULT;
676 mutex_lock(&dev->struct_mutex);
677 goto err;
678 }
679
680 reloc_offset[i] = total;
681 total += exec[i].relocation_count;
682 }
683
684 ret = i915_mutex_lock_interruptible(dev);
685 if (ret) {
686 mutex_lock(&dev->struct_mutex);
687 goto err;
688 }
689
690
691 eb_reset(eb);
692 for (i = 0; i < count; i++) {
693 struct drm_i915_gem_object *obj;
694
695 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
696 exec[i].handle));
697 if (obj == NULL) {
698 DRM_ERROR("Invalid object handle %d at index %d\n",
699 exec[i].handle, i);
700 ret = -ENOENT;
701 goto err;
702 }
703
704 list_add_tail(&obj->exec_list, objects);
705 obj->exec_handle = exec[i].handle;
706 obj->exec_entry = &exec[i];
707 eb_add_object(eb, obj);
708 }
709
710 ret = i915_gem_execbuffer_reserve(ring, file, objects);
711 if (ret)
712 goto err;
713
714 list_for_each_entry(obj, objects, exec_list) {
715 int offset = obj->exec_entry - exec;
716 ret = i915_gem_execbuffer_relocate_object_slow(obj, eb,
717 reloc + reloc_offset[offset]);
718 if (ret)
719 goto err;
720 }
721
722
723
724
725
726
727
728err:
729 drm_free_large(reloc);
730 drm_free_large(reloc_offset);
731 return ret;
732}
733
734static int
735i915_gem_execbuffer_flush(struct drm_device *dev,
736 uint32_t invalidate_domains,
737 uint32_t flush_domains,
738 uint32_t flush_rings)
739{
740 drm_i915_private_t *dev_priv = dev->dev_private;
741 int i, ret;
742
743 if (flush_domains & I915_GEM_DOMAIN_CPU)
744 intel_gtt_chipset_flush();
745
746 if (flush_domains & I915_GEM_DOMAIN_GTT)
747 wmb();
748
749 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
750 for (i = 0; i < I915_NUM_RINGS; i++)
751 if (flush_rings & (1 << i)) {
752 ret = i915_gem_flush_ring(dev,
753 &dev_priv->ring[i],
754 invalidate_domains,
755 flush_domains);
756 if (ret)
757 return ret;
758 }
759 }
760
761 return 0;
762}
763
764static int
765i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
766 struct intel_ring_buffer *to)
767{
768 struct intel_ring_buffer *from = obj->ring;
769 u32 seqno;
770 int ret, idx;
771
772 if (from == NULL || to == from)
773 return 0;
774
775
776 if (INTEL_INFO(obj->base.dev)->gen < 6 || !i915_semaphores)
777 return i915_gem_object_wait_rendering(obj, true);
778
779 idx = intel_ring_sync_index(from, to);
780
781 seqno = obj->last_rendering_seqno;
782 if (seqno <= from->sync_seqno[idx])
783 return 0;
784
785 if (seqno == from->outstanding_lazy_request) {
786 struct drm_i915_gem_request *request;
787
788 request = kzalloc(sizeof(*request), GFP_KERNEL);
789 if (request == NULL)
790 return -ENOMEM;
791
792 ret = i915_add_request(obj->base.dev, NULL, request, from);
793 if (ret) {
794 kfree(request);
795 return ret;
796 }
797
798 seqno = request->seqno;
799 }
800
801 from->sync_seqno[idx] = seqno;
802 return intel_ring_sync(to, from, seqno - 1);
803}
804
805static int
806i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
807 struct list_head *objects)
808{
809 struct drm_i915_gem_object *obj;
810 struct change_domains cd;
811 int ret;
812
813 cd.invalidate_domains = 0;
814 cd.flush_domains = 0;
815 cd.flush_rings = 0;
816 list_for_each_entry(obj, objects, exec_list)
817 i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
818
819 if (cd.invalidate_domains | cd.flush_domains) {
820#if WATCH_EXEC
821 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
822 __func__,
823 cd.invalidate_domains,
824 cd.flush_domains);
825#endif
826 ret = i915_gem_execbuffer_flush(ring->dev,
827 cd.invalidate_domains,
828 cd.flush_domains,
829 cd.flush_rings);
830 if (ret)
831 return ret;
832 }
833
834 list_for_each_entry(obj, objects, exec_list) {
835 ret = i915_gem_execbuffer_sync_rings(obj, ring);
836 if (ret)
837 return ret;
838 }
839
840 return 0;
841}
842
843static bool
844i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
845{
846 return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0;
847}
848
849static int
850validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
851 int count)
852{
853 int i;
854
855 for (i = 0; i < count; i++) {
856 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
857 int length;
858
859
860 if (exec[i].relocation_count >
861 INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
862 return -EINVAL;
863
864 length = exec[i].relocation_count *
865 sizeof(struct drm_i915_gem_relocation_entry);
866 if (!access_ok(VERIFY_READ, ptr, length))
867 return -EFAULT;
868
869
870 if (!access_ok(VERIFY_WRITE, ptr, length))
871 return -EFAULT;
872
873 if (fault_in_pages_readable(ptr, length))
874 return -EFAULT;
875 }
876
877 return 0;
878}
879
880static int
881i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring,
882 struct list_head *objects)
883{
884 struct drm_i915_gem_object *obj;
885 int flips;
886
887
888
889
890
891 flips = 0;
892 list_for_each_entry(obj, objects, exec_list) {
893 if (obj->base.write_domain)
894 flips |= atomic_read(&obj->pending_flip);
895 }
896 if (flips) {
897 int plane, flip_mask, ret;
898
899 for (plane = 0; flips >> plane; plane++) {
900 if (((flips >> plane) & 1) == 0)
901 continue;
902
903 if (plane)
904 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
905 else
906 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
907
908 ret = intel_ring_begin(ring, 2);
909 if (ret)
910 return ret;
911
912 intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
913 intel_ring_emit(ring, MI_NOOP);
914 intel_ring_advance(ring);
915 }
916 }
917
918 return 0;
919}
920
921static void
922i915_gem_execbuffer_move_to_active(struct list_head *objects,
923 struct intel_ring_buffer *ring,
924 u32 seqno)
925{
926 struct drm_i915_gem_object *obj;
927
928 list_for_each_entry(obj, objects, exec_list) {
929 obj->base.read_domains = obj->base.pending_read_domains;
930 obj->base.write_domain = obj->base.pending_write_domain;
931 obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
932
933 i915_gem_object_move_to_active(obj, ring, seqno);
934 if (obj->base.write_domain) {
935 obj->dirty = 1;
936 obj->pending_gpu_write = true;
937 list_move_tail(&obj->gpu_write_list,
938 &ring->gpu_write_list);
939 intel_mark_busy(ring->dev, obj);
940 }
941
942 trace_i915_gem_object_change_domain(obj,
943 obj->base.read_domains,
944 obj->base.write_domain);
945 }
946}
947
948static void
949i915_gem_execbuffer_retire_commands(struct drm_device *dev,
950 struct drm_file *file,
951 struct intel_ring_buffer *ring)
952{
953 struct drm_i915_gem_request *request;
954 u32 invalidate;
955
956
957
958
959
960
961
962 invalidate = I915_GEM_DOMAIN_COMMAND;
963 if (INTEL_INFO(dev)->gen >= 4)
964 invalidate |= I915_GEM_DOMAIN_SAMPLER;
965 if (ring->flush(ring, invalidate, 0)) {
966 i915_gem_next_request_seqno(dev, ring);
967 return;
968 }
969
970
971 request = kzalloc(sizeof(*request), GFP_KERNEL);
972 if (request == NULL || i915_add_request(dev, file, request, ring)) {
973 i915_gem_next_request_seqno(dev, ring);
974 kfree(request);
975 }
976}
977
978static int
979i915_gem_do_execbuffer(struct drm_device *dev, void *data,
980 struct drm_file *file,
981 struct drm_i915_gem_execbuffer2 *args,
982 struct drm_i915_gem_exec_object2 *exec)
983{
984 drm_i915_private_t *dev_priv = dev->dev_private;
985 struct list_head objects;
986 struct eb_objects *eb;
987 struct drm_i915_gem_object *batch_obj;
988 struct drm_clip_rect *cliprects = NULL;
989 struct intel_ring_buffer *ring;
990 u32 exec_start, exec_len;
991 u32 seqno;
992 int ret, mode, i;
993
994 if (!i915_gem_check_execbuffer(args)) {
995 DRM_ERROR("execbuf with invalid offset/length\n");
996 return -EINVAL;
997 }
998
999 ret = validate_exec_list(exec, args->buffer_count);
1000 if (ret)
1001 return ret;
1002
1003#if WATCH_EXEC
1004 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1005 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1006#endif
1007 switch (args->flags & I915_EXEC_RING_MASK) {
1008 case I915_EXEC_DEFAULT:
1009 case I915_EXEC_RENDER:
1010 ring = &dev_priv->ring[RCS];
1011 break;
1012 case I915_EXEC_BSD:
1013 if (!HAS_BSD(dev)) {
1014 DRM_ERROR("execbuf with invalid ring (BSD)\n");
1015 return -EINVAL;
1016 }
1017 ring = &dev_priv->ring[VCS];
1018 break;
1019 case I915_EXEC_BLT:
1020 if (!HAS_BLT(dev)) {
1021 DRM_ERROR("execbuf with invalid ring (BLT)\n");
1022 return -EINVAL;
1023 }
1024 ring = &dev_priv->ring[BCS];
1025 break;
1026 default:
1027 DRM_ERROR("execbuf with unknown ring: %d\n",
1028 (int)(args->flags & I915_EXEC_RING_MASK));
1029 return -EINVAL;
1030 }
1031
1032 mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1033 switch (mode) {
1034 case I915_EXEC_CONSTANTS_REL_GENERAL:
1035 case I915_EXEC_CONSTANTS_ABSOLUTE:
1036 case I915_EXEC_CONSTANTS_REL_SURFACE:
1037 if (ring == &dev_priv->ring[RCS] &&
1038 mode != dev_priv->relative_constants_mode) {
1039 if (INTEL_INFO(dev)->gen < 4)
1040 return -EINVAL;
1041
1042 if (INTEL_INFO(dev)->gen > 5 &&
1043 mode == I915_EXEC_CONSTANTS_REL_SURFACE)
1044 return -EINVAL;
1045
1046 ret = intel_ring_begin(ring, 4);
1047 if (ret)
1048 return ret;
1049
1050 intel_ring_emit(ring, MI_NOOP);
1051 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1052 intel_ring_emit(ring, INSTPM);
1053 intel_ring_emit(ring,
1054 I915_EXEC_CONSTANTS_MASK << 16 | mode);
1055 intel_ring_advance(ring);
1056
1057 dev_priv->relative_constants_mode = mode;
1058 }
1059 break;
1060 default:
1061 DRM_ERROR("execbuf with unknown constants: %d\n", mode);
1062 return -EINVAL;
1063 }
1064
1065 if (args->buffer_count < 1) {
1066 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1067 return -EINVAL;
1068 }
1069
1070 if (args->num_cliprects != 0) {
1071 if (ring != &dev_priv->ring[RCS]) {
1072 DRM_ERROR("clip rectangles are only valid with the render ring\n");
1073 return -EINVAL;
1074 }
1075
1076 cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects),
1077 GFP_KERNEL);
1078 if (cliprects == NULL) {
1079 ret = -ENOMEM;
1080 goto pre_mutex_err;
1081 }
1082
1083 if (copy_from_user(cliprects,
1084 (struct drm_clip_rect __user *)(uintptr_t)
1085 args->cliprects_ptr,
1086 sizeof(*cliprects)*args->num_cliprects)) {
1087 ret = -EFAULT;
1088 goto pre_mutex_err;
1089 }
1090 }
1091
1092 ret = i915_mutex_lock_interruptible(dev);
1093 if (ret)
1094 goto pre_mutex_err;
1095
1096 if (dev_priv->mm.suspended) {
1097 mutex_unlock(&dev->struct_mutex);
1098 ret = -EBUSY;
1099 goto pre_mutex_err;
1100 }
1101
1102 eb = eb_create(args->buffer_count);
1103 if (eb == NULL) {
1104 mutex_unlock(&dev->struct_mutex);
1105 ret = -ENOMEM;
1106 goto pre_mutex_err;
1107 }
1108
1109
1110 INIT_LIST_HEAD(&objects);
1111 for (i = 0; i < args->buffer_count; i++) {
1112 struct drm_i915_gem_object *obj;
1113
1114 obj = to_intel_bo(drm_gem_object_lookup(dev, file,
1115 exec[i].handle));
1116 if (obj == NULL) {
1117 DRM_ERROR("Invalid object handle %d at index %d\n",
1118 exec[i].handle, i);
1119
1120 ret = -ENOENT;
1121 goto err;
1122 }
1123
1124 if (!list_empty(&obj->exec_list)) {
1125 DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n",
1126 obj, exec[i].handle, i);
1127 ret = -EINVAL;
1128 goto err;
1129 }
1130
1131 list_add_tail(&obj->exec_list, &objects);
1132 obj->exec_handle = exec[i].handle;
1133 obj->exec_entry = &exec[i];
1134 eb_add_object(eb, obj);
1135 }
1136
1137
1138 batch_obj = list_entry(objects.prev,
1139 struct drm_i915_gem_object,
1140 exec_list);
1141
1142
1143 ret = i915_gem_execbuffer_reserve(ring, file, &objects);
1144 if (ret)
1145 goto err;
1146
1147
1148 ret = i915_gem_execbuffer_relocate(dev, eb, &objects);
1149 if (ret) {
1150 if (ret == -EFAULT) {
1151 ret = i915_gem_execbuffer_relocate_slow(dev, file, ring,
1152 &objects, eb,
1153 exec,
1154 args->buffer_count);
1155 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1156 }
1157 if (ret)
1158 goto err;
1159 }
1160
1161
1162 if (batch_obj->base.pending_write_domain) {
1163 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
1164 ret = -EINVAL;
1165 goto err;
1166 }
1167 batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1168
1169 ret = i915_gem_execbuffer_move_to_gpu(ring, &objects);
1170 if (ret)
1171 goto err;
1172
1173 ret = i915_gem_execbuffer_wait_for_flips(ring, &objects);
1174 if (ret)
1175 goto err;
1176
1177 seqno = i915_gem_next_request_seqno(dev, ring);
1178 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) {
1179 if (seqno < ring->sync_seqno[i]) {
1180
1181
1182
1183
1184 ret = i915_gpu_idle(dev);
1185 if (ret)
1186 goto err;
1187
1188 BUG_ON(ring->sync_seqno[i]);
1189 }
1190 }
1191
1192 exec_start = batch_obj->gtt_offset + args->batch_start_offset;
1193 exec_len = args->batch_len;
1194 if (cliprects) {
1195 for (i = 0; i < args->num_cliprects; i++) {
1196 ret = i915_emit_box(dev, &cliprects[i],
1197 args->DR1, args->DR4);
1198 if (ret)
1199 goto err;
1200
1201 ret = ring->dispatch_execbuffer(ring,
1202 exec_start, exec_len);
1203 if (ret)
1204 goto err;
1205 }
1206 } else {
1207 ret = ring->dispatch_execbuffer(ring, exec_start, exec_len);
1208 if (ret)
1209 goto err;
1210 }
1211
1212 i915_gem_execbuffer_move_to_active(&objects, ring, seqno);
1213 i915_gem_execbuffer_retire_commands(dev, file, ring);
1214
1215err:
1216 eb_destroy(eb);
1217 while (!list_empty(&objects)) {
1218 struct drm_i915_gem_object *obj;
1219
1220 obj = list_first_entry(&objects,
1221 struct drm_i915_gem_object,
1222 exec_list);
1223 list_del_init(&obj->exec_list);
1224 drm_gem_object_unreference(&obj->base);
1225 }
1226
1227 mutex_unlock(&dev->struct_mutex);
1228
1229pre_mutex_err:
1230 kfree(cliprects);
1231 return ret;
1232}
1233
1234
1235
1236
1237
1238int
1239i915_gem_execbuffer(struct drm_device *dev, void *data,
1240 struct drm_file *file)
1241{
1242 struct drm_i915_gem_execbuffer *args = data;
1243 struct drm_i915_gem_execbuffer2 exec2;
1244 struct drm_i915_gem_exec_object *exec_list = NULL;
1245 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1246 int ret, i;
1247
1248#if WATCH_EXEC
1249 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1250 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1251#endif
1252
1253 if (args->buffer_count < 1) {
1254 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1255 return -EINVAL;
1256 }
1257
1258
1259 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1260 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1261 if (exec_list == NULL || exec2_list == NULL) {
1262 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1263 args->buffer_count);
1264 drm_free_large(exec_list);
1265 drm_free_large(exec2_list);
1266 return -ENOMEM;
1267 }
1268 ret = copy_from_user(exec_list,
1269 (struct drm_i915_relocation_entry __user *)
1270 (uintptr_t) args->buffers_ptr,
1271 sizeof(*exec_list) * args->buffer_count);
1272 if (ret != 0) {
1273 DRM_ERROR("copy %d exec entries failed %d\n",
1274 args->buffer_count, ret);
1275 drm_free_large(exec_list);
1276 drm_free_large(exec2_list);
1277 return -EFAULT;
1278 }
1279
1280 for (i = 0; i < args->buffer_count; i++) {
1281 exec2_list[i].handle = exec_list[i].handle;
1282 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1283 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1284 exec2_list[i].alignment = exec_list[i].alignment;
1285 exec2_list[i].offset = exec_list[i].offset;
1286 if (INTEL_INFO(dev)->gen < 4)
1287 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1288 else
1289 exec2_list[i].flags = 0;
1290 }
1291
1292 exec2.buffers_ptr = args->buffers_ptr;
1293 exec2.buffer_count = args->buffer_count;
1294 exec2.batch_start_offset = args->batch_start_offset;
1295 exec2.batch_len = args->batch_len;
1296 exec2.DR1 = args->DR1;
1297 exec2.DR4 = args->DR4;
1298 exec2.num_cliprects = args->num_cliprects;
1299 exec2.cliprects_ptr = args->cliprects_ptr;
1300 exec2.flags = I915_EXEC_RENDER;
1301
1302 ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1303 if (!ret) {
1304
1305 for (i = 0; i < args->buffer_count; i++)
1306 exec_list[i].offset = exec2_list[i].offset;
1307
1308 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1309 (uintptr_t) args->buffers_ptr,
1310 exec_list,
1311 sizeof(*exec_list) * args->buffer_count);
1312 if (ret) {
1313 ret = -EFAULT;
1314 DRM_ERROR("failed to copy %d exec entries "
1315 "back to user (%d)\n",
1316 args->buffer_count, ret);
1317 }
1318 }
1319
1320 drm_free_large(exec_list);
1321 drm_free_large(exec2_list);
1322 return ret;
1323}
1324
1325int
1326i915_gem_execbuffer2(struct drm_device *dev, void *data,
1327 struct drm_file *file)
1328{
1329 struct drm_i915_gem_execbuffer2 *args = data;
1330 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1331 int ret;
1332
1333#if WATCH_EXEC
1334 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1335 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1336#endif
1337
1338 if (args->buffer_count < 1) {
1339 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
1340 return -EINVAL;
1341 }
1342
1343 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1344 if (exec2_list == NULL) {
1345 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
1346 args->buffer_count);
1347 return -ENOMEM;
1348 }
1349 ret = copy_from_user(exec2_list,
1350 (struct drm_i915_relocation_entry __user *)
1351 (uintptr_t) args->buffers_ptr,
1352 sizeof(*exec2_list) * args->buffer_count);
1353 if (ret != 0) {
1354 DRM_ERROR("copy %d exec entries failed %d\n",
1355 args->buffer_count, ret);
1356 drm_free_large(exec2_list);
1357 return -EFAULT;
1358 }
1359
1360 ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1361 if (!ret) {
1362
1363 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1364 (uintptr_t) args->buffers_ptr,
1365 exec2_list,
1366 sizeof(*exec2_list) * args->buffer_count);
1367 if (ret) {
1368 ret = -EFAULT;
1369 DRM_ERROR("failed to copy %d exec entries "
1370 "back to user (%d)\n",
1371 args->buffer_count, ret);
1372 }
1373 }
1374
1375 drm_free_large(exec2_list);
1376 return ret;
1377}
1378