1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/prefetch.h>
26#include <linux/dma-fence-array.h>
27#include <linux/sched.h>
28#include <linux/sched/clock.h>
29#include <linux/sched/signal.h>
30
31#include "i915_drv.h"
32
33static const char *i915_fence_get_driver_name(struct dma_fence *fence)
34{
35 return "i915";
36}
37
38static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
39{
40
41
42
43
44
45
46
47
48
49 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
50 return "signaled";
51
52 return to_request(fence)->timeline->common->name;
53}
54
55static bool i915_fence_signaled(struct dma_fence *fence)
56{
57 return i915_request_completed(to_request(fence));
58}
59
60static bool i915_fence_enable_signaling(struct dma_fence *fence)
61{
62 if (i915_fence_signaled(fence))
63 return false;
64
65 intel_engine_enable_signaling(to_request(fence), true);
66 return !i915_fence_signaled(fence);
67}
68
69static signed long i915_fence_wait(struct dma_fence *fence,
70 bool interruptible,
71 signed long timeout)
72{
73 return i915_request_wait(to_request(fence), interruptible, timeout);
74}
75
76static void i915_fence_release(struct dma_fence *fence)
77{
78 struct i915_request *rq = to_request(fence);
79
80
81
82
83
84
85
86
87 i915_sw_fence_fini(&rq->submit);
88
89 kmem_cache_free(rq->i915->requests, rq);
90}
91
92const struct dma_fence_ops i915_fence_ops = {
93 .get_driver_name = i915_fence_get_driver_name,
94 .get_timeline_name = i915_fence_get_timeline_name,
95 .enable_signaling = i915_fence_enable_signaling,
96 .signaled = i915_fence_signaled,
97 .wait = i915_fence_wait,
98 .release = i915_fence_release,
99};
100
101static inline void
102i915_request_remove_from_client(struct i915_request *request)
103{
104 struct drm_i915_file_private *file_priv;
105
106 file_priv = request->file_priv;
107 if (!file_priv)
108 return;
109
110 spin_lock(&file_priv->mm.lock);
111 if (request->file_priv) {
112 list_del(&request->client_link);
113 request->file_priv = NULL;
114 }
115 spin_unlock(&file_priv->mm.lock);
116}
117
118static struct i915_dependency *
119i915_dependency_alloc(struct drm_i915_private *i915)
120{
121 return kmem_cache_alloc(i915->dependencies, GFP_KERNEL);
122}
123
124static void
125i915_dependency_free(struct drm_i915_private *i915,
126 struct i915_dependency *dep)
127{
128 kmem_cache_free(i915->dependencies, dep);
129}
130
131static void
132__i915_priotree_add_dependency(struct i915_priotree *pt,
133 struct i915_priotree *signal,
134 struct i915_dependency *dep,
135 unsigned long flags)
136{
137 INIT_LIST_HEAD(&dep->dfs_link);
138 list_add(&dep->wait_link, &signal->waiters_list);
139 list_add(&dep->signal_link, &pt->signalers_list);
140 dep->signaler = signal;
141 dep->flags = flags;
142}
143
144static int
145i915_priotree_add_dependency(struct drm_i915_private *i915,
146 struct i915_priotree *pt,
147 struct i915_priotree *signal)
148{
149 struct i915_dependency *dep;
150
151 dep = i915_dependency_alloc(i915);
152 if (!dep)
153 return -ENOMEM;
154
155 __i915_priotree_add_dependency(pt, signal, dep, I915_DEPENDENCY_ALLOC);
156 return 0;
157}
158
159static void
160i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
161{
162 struct i915_dependency *dep, *next;
163
164 GEM_BUG_ON(!list_empty(&pt->link));
165
166
167
168
169
170
171
172 list_for_each_entry_safe(dep, next, &pt->signalers_list, signal_link) {
173 GEM_BUG_ON(!i915_priotree_signaled(dep->signaler));
174 GEM_BUG_ON(!list_empty(&dep->dfs_link));
175
176 list_del(&dep->wait_link);
177 if (dep->flags & I915_DEPENDENCY_ALLOC)
178 i915_dependency_free(i915, dep);
179 }
180
181
182 list_for_each_entry_safe(dep, next, &pt->waiters_list, wait_link) {
183 GEM_BUG_ON(dep->signaler != pt);
184 GEM_BUG_ON(!list_empty(&dep->dfs_link));
185
186 list_del(&dep->signal_link);
187 if (dep->flags & I915_DEPENDENCY_ALLOC)
188 i915_dependency_free(i915, dep);
189 }
190}
191
192static void
193i915_priotree_init(struct i915_priotree *pt)
194{
195 INIT_LIST_HEAD(&pt->signalers_list);
196 INIT_LIST_HEAD(&pt->waiters_list);
197 INIT_LIST_HEAD(&pt->link);
198 pt->priority = I915_PRIORITY_INVALID;
199}
200
201static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
202{
203 struct intel_engine_cs *engine;
204 enum intel_engine_id id;
205 int ret;
206
207
208 ret = i915_gem_wait_for_idle(i915,
209 I915_WAIT_INTERRUPTIBLE |
210 I915_WAIT_LOCKED);
211 if (ret)
212 return ret;
213
214
215 for_each_engine(engine, i915, id) {
216 struct i915_gem_timeline *timeline;
217 struct intel_timeline *tl = engine->timeline;
218
219 if (!i915_seqno_passed(seqno, tl->seqno)) {
220
221 intel_engine_disarm_breadcrumbs(engine);
222 GEM_BUG_ON(!list_empty(&engine->breadcrumbs.signals));
223 }
224
225
226 GEM_BUG_ON(!intel_engine_is_idle(engine));
227 GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
228
229
230 intel_engine_init_global_seqno(engine, seqno);
231 tl->seqno = seqno;
232
233 list_for_each_entry(timeline, &i915->gt.timelines, link)
234 memset(timeline->engine[id].global_sync, 0,
235 sizeof(timeline->engine[id].global_sync));
236 }
237
238 return 0;
239}
240
241int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
242{
243 struct drm_i915_private *i915 = to_i915(dev);
244
245 lockdep_assert_held(&i915->drm.struct_mutex);
246
247 if (seqno == 0)
248 return -EINVAL;
249
250
251 return reset_all_global_seqno(i915, seqno - 1);
252}
253
254static void mark_busy(struct drm_i915_private *i915)
255{
256 if (i915->gt.awake)
257 return;
258
259 GEM_BUG_ON(!i915->gt.active_requests);
260
261 intel_runtime_pm_get_noresume(i915);
262
263
264
265
266
267
268
269
270
271
272
273
274 intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
275
276 i915->gt.awake = true;
277 if (unlikely(++i915->gt.epoch == 0))
278 i915->gt.epoch = 1;
279
280 intel_enable_gt_powersave(i915);
281 i915_update_gfx_val(i915);
282 if (INTEL_GEN(i915) >= 6)
283 gen6_rps_busy(i915);
284 i915_pmu_gt_unparked(i915);
285
286 intel_engines_unpark(i915);
287
288 i915_queue_hangcheck(i915);
289
290 queue_delayed_work(i915->wq,
291 &i915->gt.retire_work,
292 round_jiffies_up_relative(HZ));
293}
294
295static int reserve_engine(struct intel_engine_cs *engine)
296{
297 struct drm_i915_private *i915 = engine->i915;
298 u32 active = ++engine->timeline->inflight_seqnos;
299 u32 seqno = engine->timeline->seqno;
300 int ret;
301
302
303 if (unlikely(add_overflows(seqno, active))) {
304 ret = reset_all_global_seqno(i915, 0);
305 if (ret) {
306 engine->timeline->inflight_seqnos--;
307 return ret;
308 }
309 }
310
311 if (!i915->gt.active_requests++)
312 mark_busy(i915);
313
314 return 0;
315}
316
317static void unreserve_engine(struct intel_engine_cs *engine)
318{
319 struct drm_i915_private *i915 = engine->i915;
320
321 if (!--i915->gt.active_requests) {
322
323 GEM_BUG_ON(!i915->gt.awake);
324 mod_delayed_work(i915->wq,
325 &i915->gt.idle_work,
326 msecs_to_jiffies(100));
327 }
328
329 GEM_BUG_ON(!engine->timeline->inflight_seqnos);
330 engine->timeline->inflight_seqnos--;
331}
332
333void i915_gem_retire_noop(struct i915_gem_active *active,
334 struct i915_request *request)
335{
336
337}
338
339static void advance_ring(struct i915_request *request)
340{
341 unsigned int tail;
342
343
344
345
346
347
348
349
350
351
352 if (list_is_last(&request->ring_link, &request->ring->request_list)) {
353
354
355
356
357
358
359
360
361 tail = READ_ONCE(request->ring->tail);
362 } else {
363 tail = request->postfix;
364 }
365 list_del(&request->ring_link);
366
367 request->ring->head = tail;
368}
369
370static void free_capture_list(struct i915_request *request)
371{
372 struct i915_capture_list *capture;
373
374 capture = request->capture_list;
375 while (capture) {
376 struct i915_capture_list *next = capture->next;
377
378 kfree(capture);
379 capture = next;
380 }
381}
382
383static void i915_request_retire(struct i915_request *request)
384{
385 struct intel_engine_cs *engine = request->engine;
386 struct i915_gem_active *active, *next;
387
388 lockdep_assert_held(&request->i915->drm.struct_mutex);
389 GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));
390 GEM_BUG_ON(!i915_request_completed(request));
391 GEM_BUG_ON(!request->i915->gt.active_requests);
392
393 trace_i915_request_retire(request);
394
395 spin_lock_irq(&engine->timeline->lock);
396 list_del_init(&request->link);
397 spin_unlock_irq(&engine->timeline->lock);
398
399 unreserve_engine(request->engine);
400 advance_ring(request);
401
402 free_capture_list(request);
403
404
405
406
407
408
409
410
411
412
413
414 list_for_each_entry_safe(active, next, &request->active_list, link) {
415
416
417
418
419
420
421
422
423
424
425 prefetchw(next);
426
427 INIT_LIST_HEAD(&active->link);
428 RCU_INIT_POINTER(active->request, NULL);
429
430 active->retire(active, request);
431 }
432
433 i915_request_remove_from_client(request);
434
435
436 atomic_dec_if_positive(&request->ctx->ban_score);
437
438
439
440
441
442
443
444
445
446 if (engine->last_retired_context)
447 engine->context_unpin(engine, engine->last_retired_context);
448 engine->last_retired_context = request->ctx;
449
450 spin_lock_irq(&request->lock);
451 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &request->fence.flags))
452 dma_fence_signal_locked(&request->fence);
453 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
454 intel_engine_cancel_signaling(request);
455 if (request->waitboost) {
456 GEM_BUG_ON(!atomic_read(&request->i915->gt_pm.rps.num_waiters));
457 atomic_dec(&request->i915->gt_pm.rps.num_waiters);
458 }
459 spin_unlock_irq(&request->lock);
460
461 i915_priotree_fini(request->i915, &request->priotree);
462 i915_request_put(request);
463}
464
465void i915_request_retire_upto(struct i915_request *rq)
466{
467 struct intel_engine_cs *engine = rq->engine;
468 struct i915_request *tmp;
469
470 lockdep_assert_held(&rq->i915->drm.struct_mutex);
471 GEM_BUG_ON(!i915_request_completed(rq));
472
473 if (list_empty(&rq->link))
474 return;
475
476 do {
477 tmp = list_first_entry(&engine->timeline->requests,
478 typeof(*tmp), link);
479
480 i915_request_retire(tmp);
481 } while (tmp != rq);
482}
483
484static u32 timeline_get_seqno(struct intel_timeline *tl)
485{
486 return ++tl->seqno;
487}
488
489void __i915_request_submit(struct i915_request *request)
490{
491 struct intel_engine_cs *engine = request->engine;
492 struct intel_timeline *timeline;
493 u32 seqno;
494
495 GEM_BUG_ON(!irqs_disabled());
496 lockdep_assert_held(&engine->timeline->lock);
497
498
499 timeline = engine->timeline;
500 GEM_BUG_ON(timeline == request->timeline);
501 GEM_BUG_ON(request->global_seqno);
502
503 seqno = timeline_get_seqno(timeline);
504 GEM_BUG_ON(!seqno);
505 GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno));
506
507
508 spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
509 request->global_seqno = seqno;
510 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
511 intel_engine_enable_signaling(request, false);
512 spin_unlock(&request->lock);
513
514 engine->emit_breadcrumb(request,
515 request->ring->vaddr + request->postfix);
516
517 spin_lock(&request->timeline->lock);
518 list_move_tail(&request->link, &timeline->requests);
519 spin_unlock(&request->timeline->lock);
520
521 trace_i915_request_execute(request);
522
523 wake_up_all(&request->execute);
524}
525
526void i915_request_submit(struct i915_request *request)
527{
528 struct intel_engine_cs *engine = request->engine;
529 unsigned long flags;
530
531
532 spin_lock_irqsave(&engine->timeline->lock, flags);
533
534 __i915_request_submit(request);
535
536 spin_unlock_irqrestore(&engine->timeline->lock, flags);
537}
538
539void __i915_request_unsubmit(struct i915_request *request)
540{
541 struct intel_engine_cs *engine = request->engine;
542 struct intel_timeline *timeline;
543
544 GEM_BUG_ON(!irqs_disabled());
545 lockdep_assert_held(&engine->timeline->lock);
546
547
548
549
550
551 GEM_BUG_ON(!request->global_seqno);
552 GEM_BUG_ON(request->global_seqno != engine->timeline->seqno);
553 GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine),
554 request->global_seqno));
555 engine->timeline->seqno--;
556
557
558 spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
559 request->global_seqno = 0;
560 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
561 intel_engine_cancel_signaling(request);
562 spin_unlock(&request->lock);
563
564
565 timeline = request->timeline;
566 GEM_BUG_ON(timeline == engine->timeline);
567
568 spin_lock(&timeline->lock);
569 list_move(&request->link, &timeline->requests);
570 spin_unlock(&timeline->lock);
571
572
573
574
575
576
577
578
579}
580
581void i915_request_unsubmit(struct i915_request *request)
582{
583 struct intel_engine_cs *engine = request->engine;
584 unsigned long flags;
585
586
587 spin_lock_irqsave(&engine->timeline->lock, flags);
588
589 __i915_request_unsubmit(request);
590
591 spin_unlock_irqrestore(&engine->timeline->lock, flags);
592}
593
594static int __i915_sw_fence_call
595submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
596{
597 struct i915_request *request =
598 container_of(fence, typeof(*request), submit);
599
600 switch (state) {
601 case FENCE_COMPLETE:
602 trace_i915_request_submit(request);
603
604
605
606
607
608
609
610
611 rcu_read_lock();
612 request->engine->submit_request(request);
613 rcu_read_unlock();
614 break;
615
616 case FENCE_FREE:
617 i915_request_put(request);
618 break;
619 }
620
621 return NOTIFY_DONE;
622}
623
624
625
626
627
628
629
630
631
632
633struct i915_request *
634i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
635{
636 struct drm_i915_private *i915 = engine->i915;
637 struct i915_request *rq;
638 struct intel_ring *ring;
639 int ret;
640
641 lockdep_assert_held(&i915->drm.struct_mutex);
642
643
644
645
646
647
648 GEM_BUG_ON(ctx == i915->preempt_context);
649
650
651
652
653
654 if (i915_terminally_wedged(&i915->gpu_error))
655 return ERR_PTR(-EIO);
656
657
658
659
660
661
662 ring = engine->context_pin(engine, ctx);
663 if (IS_ERR(ring))
664 return ERR_CAST(ring);
665 GEM_BUG_ON(!ring);
666
667 ret = reserve_engine(engine);
668 if (ret)
669 goto err_unpin;
670
671 ret = intel_ring_wait_for_space(ring, MIN_SPACE_FOR_ADD_REQUEST);
672 if (ret)
673 goto err_unreserve;
674
675
676 rq = list_first_entry_or_null(&engine->timeline->requests,
677 typeof(*rq), link);
678 if (rq && i915_request_completed(rq))
679 i915_request_retire(rq);
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710 rq = kmem_cache_alloc(i915->requests,
711 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
712 if (unlikely(!rq)) {
713
714 ret = i915_gem_wait_for_idle(i915,
715 I915_WAIT_LOCKED |
716 I915_WAIT_INTERRUPTIBLE);
717 if (ret)
718 goto err_unreserve;
719
720
721
722
723
724
725
726
727
728 kmem_cache_shrink(i915->requests);
729 rcu_barrier();
730
731 rq = kmem_cache_alloc(i915->requests, GFP_KERNEL);
732 if (!rq) {
733 ret = -ENOMEM;
734 goto err_unreserve;
735 }
736 }
737
738 rq->timeline = i915_gem_context_lookup_timeline(ctx, engine);
739 GEM_BUG_ON(rq->timeline == engine->timeline);
740
741 spin_lock_init(&rq->lock);
742 dma_fence_init(&rq->fence,
743 &i915_fence_ops,
744 &rq->lock,
745 rq->timeline->fence_context,
746 timeline_get_seqno(rq->timeline));
747
748
749 i915_sw_fence_init(&i915_request_get(rq)->submit, submit_notify);
750 init_waitqueue_head(&rq->execute);
751
752 i915_priotree_init(&rq->priotree);
753
754 INIT_LIST_HEAD(&rq->active_list);
755 rq->i915 = i915;
756 rq->engine = engine;
757 rq->ctx = ctx;
758 rq->ring = ring;
759
760
761 rq->global_seqno = 0;
762 rq->signaling.wait.seqno = 0;
763 rq->file_priv = NULL;
764 rq->batch = NULL;
765 rq->capture_list = NULL;
766 rq->waitboost = false;
767
768
769
770
771
772
773
774
775 rq->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
776 GEM_BUG_ON(rq->reserved_space < engine->emit_breadcrumb_sz);
777
778
779
780
781
782
783
784 rq->head = rq->ring->emit;
785
786
787 ret = engine->emit_flush(rq, EMIT_INVALIDATE);
788 if (ret)
789 goto err_unwind;
790
791 ret = engine->request_alloc(rq);
792 if (ret)
793 goto err_unwind;
794
795
796 GEM_BUG_ON(rq->timeline->seqno != rq->fence.seqno);
797 return rq;
798
799err_unwind:
800 rq->ring->emit = rq->head;
801
802
803 GEM_BUG_ON(!list_empty(&rq->active_list));
804 GEM_BUG_ON(!list_empty(&rq->priotree.signalers_list));
805 GEM_BUG_ON(!list_empty(&rq->priotree.waiters_list));
806
807 kmem_cache_free(i915->requests, rq);
808err_unreserve:
809 unreserve_engine(engine);
810err_unpin:
811 engine->context_unpin(engine, ctx);
812 return ERR_PTR(ret);
813}
814
815static int
816i915_request_await_request(struct i915_request *to, struct i915_request *from)
817{
818 int ret;
819
820 GEM_BUG_ON(to == from);
821 GEM_BUG_ON(to->timeline == from->timeline);
822
823 if (i915_request_completed(from))
824 return 0;
825
826 if (to->engine->schedule) {
827 ret = i915_priotree_add_dependency(to->i915,
828 &to->priotree,
829 &from->priotree);
830 if (ret < 0)
831 return ret;
832 }
833
834 if (to->engine == from->engine) {
835 ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
836 &from->submit,
837 I915_FENCE_GFP);
838 return ret < 0 ? ret : 0;
839 }
840
841 if (to->engine->semaphore.sync_to) {
842 u32 seqno;
843
844 GEM_BUG_ON(!from->engine->semaphore.signal);
845
846 seqno = i915_request_global_seqno(from);
847 if (!seqno)
848 goto await_dma_fence;
849
850 if (seqno <= to->timeline->global_sync[from->engine->id])
851 return 0;
852
853 trace_i915_gem_ring_sync_to(to, from);
854 ret = to->engine->semaphore.sync_to(to, from);
855 if (ret)
856 return ret;
857
858 to->timeline->global_sync[from->engine->id] = seqno;
859 return 0;
860 }
861
862await_dma_fence:
863 ret = i915_sw_fence_await_dma_fence(&to->submit,
864 &from->fence, 0,
865 I915_FENCE_GFP);
866 return ret < 0 ? ret : 0;
867}
868
869int
870i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
871{
872 struct dma_fence **child = &fence;
873 unsigned int nchild = 1;
874 int ret;
875
876
877
878
879
880
881
882
883
884 if (dma_fence_is_array(fence)) {
885 struct dma_fence_array *array = to_dma_fence_array(fence);
886
887 child = array->fences;
888 nchild = array->num_fences;
889 GEM_BUG_ON(!nchild);
890 }
891
892 do {
893 fence = *child++;
894 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
895 continue;
896
897
898
899
900
901
902 if (fence->context == rq->fence.context)
903 continue;
904
905
906 if (fence->context != rq->i915->mm.unordered_timeline &&
907 intel_timeline_sync_is_later(rq->timeline, fence))
908 continue;
909
910 if (dma_fence_is_i915(fence))
911 ret = i915_request_await_request(rq, to_request(fence));
912 else
913 ret = i915_sw_fence_await_dma_fence(&rq->submit, fence,
914 I915_FENCE_TIMEOUT,
915 I915_FENCE_GFP);
916 if (ret < 0)
917 return ret;
918
919
920 if (fence->context != rq->i915->mm.unordered_timeline)
921 intel_timeline_sync_set(rq->timeline, fence);
922 } while (--nchild);
923
924 return 0;
925}
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947int
948i915_request_await_object(struct i915_request *to,
949 struct drm_i915_gem_object *obj,
950 bool write)
951{
952 struct dma_fence *excl;
953 int ret = 0;
954
955 if (write) {
956 struct dma_fence **shared;
957 unsigned int count, i;
958
959 ret = reservation_object_get_fences_rcu(obj->resv,
960 &excl, &count, &shared);
961 if (ret)
962 return ret;
963
964 for (i = 0; i < count; i++) {
965 ret = i915_request_await_dma_fence(to, shared[i]);
966 if (ret)
967 break;
968
969 dma_fence_put(shared[i]);
970 }
971
972 for (; i < count; i++)
973 dma_fence_put(shared[i]);
974 kfree(shared);
975 } else {
976 excl = reservation_object_get_excl_rcu(obj->resv);
977 }
978
979 if (excl) {
980 if (ret == 0)
981 ret = i915_request_await_dma_fence(to, excl);
982
983 dma_fence_put(excl);
984 }
985
986 return ret;
987}
988
989
990
991
992
993
994void __i915_request_add(struct i915_request *request, bool flush_caches)
995{
996 struct intel_engine_cs *engine = request->engine;
997 struct intel_ring *ring = request->ring;
998 struct intel_timeline *timeline = request->timeline;
999 struct i915_request *prev;
1000 u32 *cs;
1001 int err;
1002
1003 lockdep_assert_held(&request->i915->drm.struct_mutex);
1004 trace_i915_request_add(request);
1005
1006
1007
1008
1009
1010
1011 GEM_BUG_ON(timeline->seqno != request->fence.seqno);
1012
1013
1014
1015
1016
1017
1018 request->reserved_space = 0;
1019
1020
1021
1022
1023
1024
1025
1026
1027 if (flush_caches) {
1028 err = engine->emit_flush(request, EMIT_FLUSH);
1029
1030
1031 WARN(err, "engine->emit_flush() failed: %d!\n", err);
1032 }
1033
1034
1035
1036
1037
1038
1039
1040 cs = intel_ring_begin(request, engine->emit_breadcrumb_sz);
1041 GEM_BUG_ON(IS_ERR(cs));
1042 request->postfix = intel_ring_offset(request, cs);
1043
1044
1045
1046
1047
1048
1049
1050
1051 prev = i915_gem_active_raw(&timeline->last_request,
1052 &request->i915->drm.struct_mutex);
1053 if (prev && !i915_request_completed(prev)) {
1054 i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
1055 &request->submitq);
1056 if (engine->schedule)
1057 __i915_priotree_add_dependency(&request->priotree,
1058 &prev->priotree,
1059 &request->dep,
1060 0);
1061 }
1062
1063 spin_lock_irq(&timeline->lock);
1064 list_add_tail(&request->link, &timeline->requests);
1065 spin_unlock_irq(&timeline->lock);
1066
1067 GEM_BUG_ON(timeline->seqno != request->fence.seqno);
1068 i915_gem_active_set(&timeline->last_request, request);
1069
1070 list_add_tail(&request->ring_link, &ring->request_list);
1071 request->emitted_jiffies = jiffies;
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084 rcu_read_lock();
1085 if (engine->schedule)
1086 engine->schedule(request, request->ctx->priority);
1087 rcu_read_unlock();
1088
1089 local_bh_disable();
1090 i915_sw_fence_commit(&request->submit);
1091 local_bh_enable();
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110 if (prev && i915_request_completed(prev))
1111 i915_request_retire_upto(prev);
1112}
1113
1114static unsigned long local_clock_us(unsigned int *cpu)
1115{
1116 unsigned long t;
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130 *cpu = get_cpu();
1131 t = local_clock() >> 10;
1132 put_cpu();
1133
1134 return t;
1135}
1136
1137static bool busywait_stop(unsigned long timeout, unsigned int cpu)
1138{
1139 unsigned int this_cpu;
1140
1141 if (time_after(local_clock_us(&this_cpu), timeout))
1142 return true;
1143
1144 return this_cpu != cpu;
1145}
1146
1147static bool __i915_spin_request(const struct i915_request *rq,
1148 u32 seqno, int state, unsigned long timeout_us)
1149{
1150 struct intel_engine_cs *engine = rq->engine;
1151 unsigned int irq, cpu;
1152
1153 GEM_BUG_ON(!seqno);
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166 if (!i915_seqno_passed(intel_engine_get_seqno(engine), seqno - 1))
1167 return false;
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180 irq = atomic_read(&engine->irq_count);
1181 timeout_us += local_clock_us(&cpu);
1182 do {
1183 if (i915_seqno_passed(intel_engine_get_seqno(engine), seqno))
1184 return seqno == i915_request_global_seqno(rq);
1185
1186
1187
1188
1189
1190
1191
1192 if (atomic_read(&engine->irq_count) != irq)
1193 break;
1194
1195 if (signal_pending_state(state, current))
1196 break;
1197
1198 if (busywait_stop(timeout_us, cpu))
1199 break;
1200
1201 cpu_relax();
1202 } while (!need_resched());
1203
1204 return false;
1205}
1206
1207static bool __i915_wait_request_check_and_reset(struct i915_request *request)
1208{
1209 if (likely(!i915_reset_handoff(&request->i915->gpu_error)))
1210 return false;
1211
1212 __set_current_state(TASK_RUNNING);
1213 i915_reset(request->i915, 0);
1214 return true;
1215}
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236long i915_request_wait(struct i915_request *rq,
1237 unsigned int flags,
1238 long timeout)
1239{
1240 const int state = flags & I915_WAIT_INTERRUPTIBLE ?
1241 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1242 wait_queue_head_t *errq = &rq->i915->gpu_error.wait_queue;
1243 DEFINE_WAIT_FUNC(reset, default_wake_function);
1244 DEFINE_WAIT_FUNC(exec, default_wake_function);
1245 struct intel_wait wait;
1246
1247 might_sleep();
1248#if IS_ENABLED(CONFIG_LOCKDEP)
1249 GEM_BUG_ON(debug_locks &&
1250 !!lockdep_is_held(&rq->i915->drm.struct_mutex) !=
1251 !!(flags & I915_WAIT_LOCKED));
1252#endif
1253 GEM_BUG_ON(timeout < 0);
1254
1255 if (i915_request_completed(rq))
1256 return timeout;
1257
1258 if (!timeout)
1259 return -ETIME;
1260
1261 trace_i915_request_wait_begin(rq, flags);
1262
1263 add_wait_queue(&rq->execute, &exec);
1264 if (flags & I915_WAIT_LOCKED)
1265 add_wait_queue(errq, &reset);
1266
1267 intel_wait_init(&wait, rq);
1268
1269restart:
1270 do {
1271 set_current_state(state);
1272 if (intel_wait_update_request(&wait, rq))
1273 break;
1274
1275 if (flags & I915_WAIT_LOCKED &&
1276 __i915_wait_request_check_and_reset(rq))
1277 continue;
1278
1279 if (signal_pending_state(state, current)) {
1280 timeout = -ERESTARTSYS;
1281 goto complete;
1282 }
1283
1284 if (!timeout) {
1285 timeout = -ETIME;
1286 goto complete;
1287 }
1288
1289 timeout = io_schedule_timeout(timeout);
1290 } while (1);
1291
1292 GEM_BUG_ON(!intel_wait_has_seqno(&wait));
1293 GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
1294
1295
1296 if (__i915_spin_request(rq, wait.seqno, state, 5))
1297 goto complete;
1298
1299 set_current_state(state);
1300 if (intel_engine_add_wait(rq->engine, &wait))
1301
1302
1303
1304
1305
1306 goto wakeup;
1307
1308 if (flags & I915_WAIT_LOCKED)
1309 __i915_wait_request_check_and_reset(rq);
1310
1311 for (;;) {
1312 if (signal_pending_state(state, current)) {
1313 timeout = -ERESTARTSYS;
1314 break;
1315 }
1316
1317 if (!timeout) {
1318 timeout = -ETIME;
1319 break;
1320 }
1321
1322 timeout = io_schedule_timeout(timeout);
1323
1324 if (intel_wait_complete(&wait) &&
1325 intel_wait_check_request(&wait, rq))
1326 break;
1327
1328 set_current_state(state);
1329
1330wakeup:
1331
1332
1333
1334
1335
1336
1337 if (__i915_request_irq_complete(rq))
1338 break;
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351 if (flags & I915_WAIT_LOCKED &&
1352 __i915_wait_request_check_and_reset(rq))
1353 continue;
1354
1355
1356 if (__i915_spin_request(rq, wait.seqno, state, 2))
1357 break;
1358
1359 if (!intel_wait_check_request(&wait, rq)) {
1360 intel_engine_remove_wait(rq->engine, &wait);
1361 goto restart;
1362 }
1363 }
1364
1365 intel_engine_remove_wait(rq->engine, &wait);
1366complete:
1367 __set_current_state(TASK_RUNNING);
1368 if (flags & I915_WAIT_LOCKED)
1369 remove_wait_queue(errq, &reset);
1370 remove_wait_queue(&rq->execute, &exec);
1371 trace_i915_request_wait_end(rq);
1372
1373 return timeout;
1374}
1375
1376static void engine_retire_requests(struct intel_engine_cs *engine)
1377{
1378 struct i915_request *request, *next;
1379 u32 seqno = intel_engine_get_seqno(engine);
1380 LIST_HEAD(retire);
1381
1382 spin_lock_irq(&engine->timeline->lock);
1383 list_for_each_entry_safe(request, next,
1384 &engine->timeline->requests, link) {
1385 if (!i915_seqno_passed(seqno, request->global_seqno))
1386 break;
1387
1388 list_move_tail(&request->link, &retire);
1389 }
1390 spin_unlock_irq(&engine->timeline->lock);
1391
1392 list_for_each_entry_safe(request, next, &retire, link)
1393 i915_request_retire(request);
1394}
1395
1396void i915_retire_requests(struct drm_i915_private *i915)
1397{
1398 struct intel_engine_cs *engine;
1399 enum intel_engine_id id;
1400
1401 lockdep_assert_held(&i915->drm.struct_mutex);
1402
1403 if (!i915->gt.active_requests)
1404 return;
1405
1406 for_each_engine(engine, i915, id)
1407 engine_retire_requests(engine);
1408}
1409
1410#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1411#include "selftests/mock_request.c"
1412#include "selftests/i915_request.c"
1413#endif
1414