1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109#include <linux/interrupt.h>
110
111#include "i915_drv.h"
112#include "i915_trace.h"
113#include "i915_vgpu.h"
114#include "gen8_engine_cs.h"
115#include "intel_breadcrumbs.h"
116#include "intel_context.h"
117#include "intel_engine_heartbeat.h"
118#include "intel_engine_pm.h"
119#include "intel_engine_stats.h"
120#include "intel_execlists_submission.h"
121#include "intel_gt.h"
122#include "intel_gt_irq.h"
123#include "intel_gt_pm.h"
124#include "intel_gt_requests.h"
125#include "intel_lrc.h"
126#include "intel_lrc_reg.h"
127#include "intel_mocs.h"
128#include "intel_reset.h"
129#include "intel_ring.h"
130#include "intel_workarounds.h"
131#include "shmem_utils.h"
132
133#define RING_EXECLIST_QFULL (1 << 0x2)
134#define RING_EXECLIST1_VALID (1 << 0x3)
135#define RING_EXECLIST0_VALID (1 << 0x4)
136#define RING_EXECLIST_ACTIVE_STATUS (3 << 0xE)
137#define RING_EXECLIST1_ACTIVE (1 << 0x11)
138#define RING_EXECLIST0_ACTIVE (1 << 0x12)
139
140#define GEN8_CTX_STATUS_IDLE_ACTIVE (1 << 0)
141#define GEN8_CTX_STATUS_PREEMPTED (1 << 1)
142#define GEN8_CTX_STATUS_ELEMENT_SWITCH (1 << 2)
143#define GEN8_CTX_STATUS_ACTIVE_IDLE (1 << 3)
144#define GEN8_CTX_STATUS_COMPLETE (1 << 4)
145#define GEN8_CTX_STATUS_LITE_RESTORE (1 << 15)
146
147#define GEN8_CTX_STATUS_COMPLETED_MASK \
148 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
149
150#define GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE (0x1)
151#define GEN12_CTX_SWITCH_DETAIL(csb_dw) ((csb_dw) & 0xF)
152#define GEN12_CSB_SW_CTX_ID_MASK GENMASK(25, 15)
153#define GEN12_IDLE_CTX_ID 0x7FF
154#define GEN12_CSB_CTX_VALID(csb_dw) \
155 (FIELD_GET(GEN12_CSB_SW_CTX_ID_MASK, csb_dw) != GEN12_IDLE_CTX_ID)
156
157#define XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE BIT(1)
158#define XEHP_CSB_SW_CTX_ID_MASK GENMASK(31, 10)
159#define XEHP_IDLE_CTX_ID 0xFFFF
160#define XEHP_CSB_CTX_VALID(csb_dw) \
161 (FIELD_GET(XEHP_CSB_SW_CTX_ID_MASK, csb_dw) != XEHP_IDLE_CTX_ID)
162
163
164#define EXECLISTS_REQUEST_SIZE 64
165
166struct virtual_engine {
167 struct intel_engine_cs base;
168 struct intel_context context;
169 struct rcu_work rcu;
170
171
172
173
174
175
176
177
178
179
180 struct i915_request *request;
181
182
183
184
185
186
187 struct ve_node {
188 struct rb_node rb;
189 int prio;
190 } nodes[I915_NUM_ENGINES];
191
192
193 unsigned int num_siblings;
194 struct intel_engine_cs *siblings[];
195};
196
197static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine)
198{
199 GEM_BUG_ON(!intel_engine_is_virtual(engine));
200 return container_of(engine, struct virtual_engine, base);
201}
202
203static struct intel_context *
204execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count);
205
206static struct i915_request *
207__active_request(const struct intel_timeline * const tl,
208 struct i915_request *rq,
209 int error)
210{
211 struct i915_request *active = rq;
212
213 list_for_each_entry_from_reverse(rq, &tl->requests, link) {
214 if (__i915_request_is_complete(rq))
215 break;
216
217 if (error) {
218 i915_request_set_error_once(rq, error);
219 __i915_request_skip(rq);
220 }
221 active = rq;
222 }
223
224 return active;
225}
226
227static struct i915_request *
228active_request(const struct intel_timeline * const tl, struct i915_request *rq)
229{
230 return __active_request(tl, rq, 0);
231}
232
233static void ring_set_paused(const struct intel_engine_cs *engine, int state)
234{
235
236
237
238
239
240
241 engine->status_page.addr[I915_GEM_HWS_PREEMPT] = state;
242 if (state)
243 wmb();
244}
245
246static struct i915_priolist *to_priolist(struct rb_node *rb)
247{
248 return rb_entry(rb, struct i915_priolist, node);
249}
250
251static int rq_prio(const struct i915_request *rq)
252{
253 return READ_ONCE(rq->sched.attr.priority);
254}
255
256static int effective_prio(const struct i915_request *rq)
257{
258 int prio = rq_prio(rq);
259
260
261
262
263
264
265
266
267
268 if (i915_request_has_nopreempt(rq))
269 prio = I915_PRIORITY_UNPREEMPTABLE;
270
271 return prio;
272}
273
274static int queue_prio(const struct i915_sched_engine *sched_engine)
275{
276 struct rb_node *rb;
277
278 rb = rb_first_cached(&sched_engine->queue);
279 if (!rb)
280 return INT_MIN;
281
282 return to_priolist(rb)->priority;
283}
284
285static int virtual_prio(const struct intel_engine_execlists *el)
286{
287 struct rb_node *rb = rb_first_cached(&el->virtual);
288
289 return rb ? rb_entry(rb, struct ve_node, rb)->prio : INT_MIN;
290}
291
292static bool need_preempt(const struct intel_engine_cs *engine,
293 const struct i915_request *rq)
294{
295 int last_prio;
296
297 if (!intel_engine_has_semaphores(engine))
298 return false;
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318 last_prio = max(effective_prio(rq), I915_PRIORITY_NORMAL - 1);
319 if (engine->sched_engine->queue_priority_hint <= last_prio)
320 return false;
321
322
323
324
325
326 if (!list_is_last(&rq->sched.link, &engine->sched_engine->requests) &&
327 rq_prio(list_next_entry(rq, sched.link)) > last_prio)
328 return true;
329
330
331
332
333
334
335
336
337
338
339
340 return max(virtual_prio(&engine->execlists),
341 queue_prio(engine->sched_engine)) > last_prio;
342}
343
344__maybe_unused static bool
345assert_priority_queue(const struct i915_request *prev,
346 const struct i915_request *next)
347{
348
349
350
351
352
353
354
355 if (i915_request_is_active(prev))
356 return true;
357
358 return rq_prio(prev) >= rq_prio(next);
359}
360
361static struct i915_request *
362__unwind_incomplete_requests(struct intel_engine_cs *engine)
363{
364 struct i915_request *rq, *rn, *active = NULL;
365 struct list_head *pl;
366 int prio = I915_PRIORITY_INVALID;
367
368 lockdep_assert_held(&engine->sched_engine->lock);
369
370 list_for_each_entry_safe_reverse(rq, rn,
371 &engine->sched_engine->requests,
372 sched.link) {
373 if (__i915_request_is_complete(rq)) {
374 list_del_init(&rq->sched.link);
375 continue;
376 }
377
378 __i915_request_unsubmit(rq);
379
380 GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
381 if (rq_prio(rq) != prio) {
382 prio = rq_prio(rq);
383 pl = i915_sched_lookup_priolist(engine->sched_engine,
384 prio);
385 }
386 GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine));
387
388 list_move(&rq->sched.link, pl);
389 set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
390
391
392 if (intel_ring_direction(rq->ring,
393 rq->tail,
394 rq->ring->tail + 8) > 0)
395 rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE;
396
397 active = rq;
398 }
399
400 return active;
401}
402
403struct i915_request *
404execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
405{
406 struct intel_engine_cs *engine =
407 container_of(execlists, typeof(*engine), execlists);
408
409 return __unwind_incomplete_requests(engine);
410}
411
412static void
413execlists_context_status_change(struct i915_request *rq, unsigned long status)
414{
415
416
417
418
419 if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
420 return;
421
422 atomic_notifier_call_chain(&rq->engine->context_status_notifier,
423 status, rq);
424}
425
426static void reset_active(struct i915_request *rq,
427 struct intel_engine_cs *engine)
428{
429 struct intel_context * const ce = rq->context;
430 u32 head;
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447 ENGINE_TRACE(engine, "{ reset rq=%llx:%lld }\n",
448 rq->fence.context, rq->fence.seqno);
449
450
451 if (__i915_request_is_complete(rq))
452 head = rq->tail;
453 else
454 head = __active_request(ce->timeline, rq, -EIO)->head;
455 head = intel_ring_wrap(ce->ring, head);
456
457
458 lrc_init_regs(ce, engine, true);
459
460
461 ce->lrc.lrca = lrc_update_regs(ce, engine, head);
462}
463
464static bool bad_request(const struct i915_request *rq)
465{
466 return rq->fence.error && i915_request_started(rq);
467}
468
469static struct intel_engine_cs *
470__execlists_schedule_in(struct i915_request *rq)
471{
472 struct intel_engine_cs * const engine = rq->engine;
473 struct intel_context * const ce = rq->context;
474
475 intel_context_get(ce);
476
477 if (unlikely(intel_context_is_closed(ce) &&
478 !intel_engine_has_heartbeat(engine)))
479 intel_context_set_banned(ce);
480
481 if (unlikely(intel_context_is_banned(ce) || bad_request(rq)))
482 reset_active(rq, engine);
483
484 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
485 lrc_check_regs(ce, engine, "before");
486
487 if (ce->tag) {
488
489 GEM_BUG_ON(ce->tag <= BITS_PER_LONG);
490 ce->lrc.ccid = ce->tag;
491 } else if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
492
493 unsigned int tag = ffs(READ_ONCE(engine->context_tag));
494
495 GEM_BUG_ON(tag == 0 || tag >= BITS_PER_LONG);
496 clear_bit(tag - 1, &engine->context_tag);
497 ce->lrc.ccid = tag << (XEHP_SW_CTX_ID_SHIFT - 32);
498
499 BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID);
500
501 } else {
502
503 unsigned int tag = __ffs(engine->context_tag);
504
505 GEM_BUG_ON(tag >= BITS_PER_LONG);
506 __clear_bit(tag, &engine->context_tag);
507 ce->lrc.ccid = (1 + tag) << (GEN11_SW_CTX_ID_SHIFT - 32);
508
509 BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID);
510 }
511
512 ce->lrc.ccid |= engine->execlists.ccid;
513
514 __intel_gt_pm_get(engine->gt);
515 if (engine->fw_domain && !engine->fw_active++)
516 intel_uncore_forcewake_get(engine->uncore, engine->fw_domain);
517 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
518 intel_engine_context_in(engine);
519
520 CE_TRACE(ce, "schedule-in, ccid:%x\n", ce->lrc.ccid);
521
522 return engine;
523}
524
525static void execlists_schedule_in(struct i915_request *rq, int idx)
526{
527 struct intel_context * const ce = rq->context;
528 struct intel_engine_cs *old;
529
530 GEM_BUG_ON(!intel_engine_pm_is_awake(rq->engine));
531 trace_i915_request_in(rq, idx);
532
533 old = ce->inflight;
534 if (!old)
535 old = __execlists_schedule_in(rq);
536 WRITE_ONCE(ce->inflight, ptr_inc(old));
537
538 GEM_BUG_ON(intel_context_inflight(ce) != rq->engine);
539}
540
541static void
542resubmit_virtual_request(struct i915_request *rq, struct virtual_engine *ve)
543{
544 struct intel_engine_cs *engine = rq->engine;
545
546 spin_lock_irq(&engine->sched_engine->lock);
547
548 clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
549 WRITE_ONCE(rq->engine, &ve->base);
550 ve->base.submit_request(rq);
551
552 spin_unlock_irq(&engine->sched_engine->lock);
553}
554
555static void kick_siblings(struct i915_request *rq, struct intel_context *ce)
556{
557 struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
558 struct intel_engine_cs *engine = rq->engine;
559
560
561
562
563
564
565
566
567 if (!list_empty(&ce->signals))
568 intel_context_remove_breadcrumbs(ce, engine->breadcrumbs);
569
570
571
572
573
574
575
576 if (i915_request_in_priority_queue(rq) &&
577 rq->execution_mask != engine->mask)
578 resubmit_virtual_request(rq, ve);
579
580 if (READ_ONCE(ve->request))
581 tasklet_hi_schedule(&ve->base.sched_engine->tasklet);
582}
583
584static void __execlists_schedule_out(struct i915_request * const rq,
585 struct intel_context * const ce)
586{
587 struct intel_engine_cs * const engine = rq->engine;
588 unsigned int ccid;
589
590
591
592
593
594
595
596 CE_TRACE(ce, "schedule-out, ccid:%x\n", ce->lrc.ccid);
597 GEM_BUG_ON(ce->inflight != engine);
598
599 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
600 lrc_check_regs(ce, engine, "after");
601
602
603
604
605
606 if (intel_timeline_is_last(ce->timeline, rq) &&
607 __i915_request_is_complete(rq))
608 intel_engine_add_retire(engine, ce->timeline);
609
610 ccid = ce->lrc.ccid;
611 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) {
612 ccid >>= XEHP_SW_CTX_ID_SHIFT - 32;
613 ccid &= XEHP_MAX_CONTEXT_HW_ID;
614 } else {
615 ccid >>= GEN11_SW_CTX_ID_SHIFT - 32;
616 ccid &= GEN12_MAX_CONTEXT_HW_ID;
617 }
618
619 if (ccid < BITS_PER_LONG) {
620 GEM_BUG_ON(ccid == 0);
621 GEM_BUG_ON(test_bit(ccid - 1, &engine->context_tag));
622 __set_bit(ccid - 1, &engine->context_tag);
623 }
624
625 lrc_update_runtime(ce);
626 intel_engine_context_out(engine);
627 execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
628 if (engine->fw_domain && !--engine->fw_active)
629 intel_uncore_forcewake_put(engine->uncore, engine->fw_domain);
630 intel_gt_pm_put_async(engine->gt);
631
632
633
634
635
636
637
638
639
640
641 if (ce->engine != engine)
642 kick_siblings(rq, ce);
643
644 WRITE_ONCE(ce->inflight, NULL);
645 intel_context_put(ce);
646}
647
648static inline void execlists_schedule_out(struct i915_request *rq)
649{
650 struct intel_context * const ce = rq->context;
651
652 trace_i915_request_out(rq);
653
654 GEM_BUG_ON(!ce->inflight);
655 ce->inflight = ptr_dec(ce->inflight);
656 if (!__intel_context_inflight_count(ce->inflight))
657 __execlists_schedule_out(rq, ce);
658
659 i915_request_put(rq);
660}
661
662static u64 execlists_update_context(struct i915_request *rq)
663{
664 struct intel_context *ce = rq->context;
665 u64 desc = ce->lrc.desc;
666 u32 tail, prev;
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685 GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != rq->ring->tail);
686 prev = rq->ring->tail;
687 tail = intel_ring_set_tail(rq->ring, rq->tail);
688 if (unlikely(intel_ring_direction(rq->ring, tail, prev) <= 0))
689 desc |= CTX_DESC_FORCE_RESTORE;
690 ce->lrc_reg_state[CTX_RING_TAIL] = tail;
691 rq->tail = rq->wa_tail;
692
693
694
695
696
697
698
699
700
701
702
703 wmb();
704
705 ce->lrc.desc &= ~CTX_DESC_FORCE_RESTORE;
706 return desc;
707}
708
709static void write_desc(struct intel_engine_execlists *execlists, u64 desc, u32 port)
710{
711 if (execlists->ctrl_reg) {
712 writel(lower_32_bits(desc), execlists->submit_reg + port * 2);
713 writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);
714 } else {
715 writel(upper_32_bits(desc), execlists->submit_reg);
716 writel(lower_32_bits(desc), execlists->submit_reg);
717 }
718}
719
720static __maybe_unused char *
721dump_port(char *buf, int buflen, const char *prefix, struct i915_request *rq)
722{
723 if (!rq)
724 return "";
725
726 snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d",
727 prefix,
728 rq->context->lrc.ccid,
729 rq->fence.context, rq->fence.seqno,
730 __i915_request_is_complete(rq) ? "!" :
731 __i915_request_has_started(rq) ? "*" :
732 "",
733 rq_prio(rq));
734
735 return buf;
736}
737
738static __maybe_unused noinline void
739trace_ports(const struct intel_engine_execlists *execlists,
740 const char *msg,
741 struct i915_request * const *ports)
742{
743 const struct intel_engine_cs *engine =
744 container_of(execlists, typeof(*engine), execlists);
745 char __maybe_unused p0[40], p1[40];
746
747 if (!ports[0])
748 return;
749
750 ENGINE_TRACE(engine, "%s { %s%s }\n", msg,
751 dump_port(p0, sizeof(p0), "", ports[0]),
752 dump_port(p1, sizeof(p1), ", ", ports[1]));
753}
754
755static bool
756reset_in_progress(const struct intel_engine_cs *engine)
757{
758 return unlikely(!__tasklet_is_enabled(&engine->sched_engine->tasklet));
759}
760
761static __maybe_unused noinline bool
762assert_pending_valid(const struct intel_engine_execlists *execlists,
763 const char *msg)
764{
765 struct intel_engine_cs *engine =
766 container_of(execlists, typeof(*engine), execlists);
767 struct i915_request * const *port, *rq, *prev = NULL;
768 struct intel_context *ce = NULL;
769 u32 ccid = -1;
770
771 trace_ports(execlists, msg, execlists->pending);
772
773
774 if (reset_in_progress(engine))
775 return true;
776
777 if (!execlists->pending[0]) {
778 GEM_TRACE_ERR("%s: Nothing pending for promotion!\n",
779 engine->name);
780 return false;
781 }
782
783 if (execlists->pending[execlists_num_ports(execlists)]) {
784 GEM_TRACE_ERR("%s: Excess pending[%d] for promotion!\n",
785 engine->name, execlists_num_ports(execlists));
786 return false;
787 }
788
789 for (port = execlists->pending; (rq = *port); port++) {
790 unsigned long flags;
791 bool ok = true;
792
793 GEM_BUG_ON(!kref_read(&rq->fence.refcount));
794 GEM_BUG_ON(!i915_request_is_active(rq));
795
796 if (ce == rq->context) {
797 GEM_TRACE_ERR("%s: Dup context:%llx in pending[%zd]\n",
798 engine->name,
799 ce->timeline->fence_context,
800 port - execlists->pending);
801 return false;
802 }
803 ce = rq->context;
804
805 if (ccid == ce->lrc.ccid) {
806 GEM_TRACE_ERR("%s: Dup ccid:%x context:%llx in pending[%zd]\n",
807 engine->name,
808 ccid, ce->timeline->fence_context,
809 port - execlists->pending);
810 return false;
811 }
812 ccid = ce->lrc.ccid;
813
814
815
816
817
818
819
820
821
822
823 if (prev && i915_request_has_sentinel(prev) &&
824 !READ_ONCE(prev->fence.error)) {
825 GEM_TRACE_ERR("%s: context:%llx after sentinel in pending[%zd]\n",
826 engine->name,
827 ce->timeline->fence_context,
828 port - execlists->pending);
829 return false;
830 }
831 prev = rq;
832
833
834
835
836
837
838 if (rq->execution_mask != engine->mask &&
839 port != execlists->pending) {
840 GEM_TRACE_ERR("%s: virtual engine:%llx not in prime position[%zd]\n",
841 engine->name,
842 ce->timeline->fence_context,
843 port - execlists->pending);
844 return false;
845 }
846
847
848 if (!spin_trylock_irqsave(&rq->lock, flags))
849 continue;
850
851 if (__i915_request_is_complete(rq))
852 goto unlock;
853
854 if (i915_active_is_idle(&ce->active) &&
855 !intel_context_is_barrier(ce)) {
856 GEM_TRACE_ERR("%s: Inactive context:%llx in pending[%zd]\n",
857 engine->name,
858 ce->timeline->fence_context,
859 port - execlists->pending);
860 ok = false;
861 goto unlock;
862 }
863
864 if (!i915_vma_is_pinned(ce->state)) {
865 GEM_TRACE_ERR("%s: Unpinned context:%llx in pending[%zd]\n",
866 engine->name,
867 ce->timeline->fence_context,
868 port - execlists->pending);
869 ok = false;
870 goto unlock;
871 }
872
873 if (!i915_vma_is_pinned(ce->ring->vma)) {
874 GEM_TRACE_ERR("%s: Unpinned ring:%llx in pending[%zd]\n",
875 engine->name,
876 ce->timeline->fence_context,
877 port - execlists->pending);
878 ok = false;
879 goto unlock;
880 }
881
882unlock:
883 spin_unlock_irqrestore(&rq->lock, flags);
884 if (!ok)
885 return false;
886 }
887
888 return ce;
889}
890
891static void execlists_submit_ports(struct intel_engine_cs *engine)
892{
893 struct intel_engine_execlists *execlists = &engine->execlists;
894 unsigned int n;
895
896 GEM_BUG_ON(!assert_pending_valid(execlists, "submit"));
897
898
899
900
901
902
903
904
905
906 GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
907
908
909
910
911
912
913
914 for (n = execlists_num_ports(execlists); n--; ) {
915 struct i915_request *rq = execlists->pending[n];
916
917 write_desc(execlists,
918 rq ? execlists_update_context(rq) : 0,
919 n);
920 }
921
922
923 if (execlists->ctrl_reg)
924 writel(EL_CTRL_LOAD, execlists->ctrl_reg);
925}
926
927static bool ctx_single_port_submission(const struct intel_context *ce)
928{
929 return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
930 intel_context_force_single_submission(ce));
931}
932
933static bool can_merge_ctx(const struct intel_context *prev,
934 const struct intel_context *next)
935{
936 if (prev != next)
937 return false;
938
939 if (ctx_single_port_submission(prev))
940 return false;
941
942 return true;
943}
944
945static unsigned long i915_request_flags(const struct i915_request *rq)
946{
947 return READ_ONCE(rq->fence.flags);
948}
949
950static bool can_merge_rq(const struct i915_request *prev,
951 const struct i915_request *next)
952{
953 GEM_BUG_ON(prev == next);
954 GEM_BUG_ON(!assert_priority_queue(prev, next));
955
956
957
958
959
960
961
962
963
964 if (__i915_request_is_complete(next))
965 return true;
966
967 if (unlikely((i915_request_flags(prev) | i915_request_flags(next)) &
968 (BIT(I915_FENCE_FLAG_NOPREEMPT) |
969 BIT(I915_FENCE_FLAG_SENTINEL))))
970 return false;
971
972 if (!can_merge_ctx(prev->context, next->context))
973 return false;
974
975 GEM_BUG_ON(i915_seqno_passed(prev->fence.seqno, next->fence.seqno));
976 return true;
977}
978
979static bool virtual_matches(const struct virtual_engine *ve,
980 const struct i915_request *rq,
981 const struct intel_engine_cs *engine)
982{
983 const struct intel_engine_cs *inflight;
984
985 if (!rq)
986 return false;
987
988 if (!(rq->execution_mask & engine->mask))
989 return false;
990
991
992
993
994
995
996
997
998
999
1000 inflight = intel_context_inflight(&ve->context);
1001 if (inflight && inflight != engine)
1002 return false;
1003
1004 return true;
1005}
1006
1007static struct virtual_engine *
1008first_virtual_engine(struct intel_engine_cs *engine)
1009{
1010 struct intel_engine_execlists *el = &engine->execlists;
1011 struct rb_node *rb = rb_first_cached(&el->virtual);
1012
1013 while (rb) {
1014 struct virtual_engine *ve =
1015 rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
1016 struct i915_request *rq = READ_ONCE(ve->request);
1017
1018
1019 if (!rq || !virtual_matches(ve, rq, engine)) {
1020 rb_erase_cached(rb, &el->virtual);
1021 RB_CLEAR_NODE(rb);
1022 rb = rb_first_cached(&el->virtual);
1023 continue;
1024 }
1025
1026 return ve;
1027 }
1028
1029 return NULL;
1030}
1031
1032static void virtual_xfer_context(struct virtual_engine *ve,
1033 struct intel_engine_cs *engine)
1034{
1035 unsigned int n;
1036
1037 if (likely(engine == ve->siblings[0]))
1038 return;
1039
1040 GEM_BUG_ON(READ_ONCE(ve->context.inflight));
1041 if (!intel_engine_has_relative_mmio(engine))
1042 lrc_update_offsets(&ve->context, engine);
1043
1044
1045
1046
1047
1048
1049
1050 for (n = 1; n < ve->num_siblings; n++) {
1051 if (ve->siblings[n] == engine) {
1052 swap(ve->siblings[n], ve->siblings[0]);
1053 break;
1054 }
1055 }
1056}
1057
1058static void defer_request(struct i915_request *rq, struct list_head * const pl)
1059{
1060 LIST_HEAD(list);
1061
1062
1063
1064
1065
1066
1067
1068
1069 do {
1070 struct i915_dependency *p;
1071
1072 GEM_BUG_ON(i915_request_is_active(rq));
1073 list_move_tail(&rq->sched.link, pl);
1074
1075 for_each_waiter(p, rq) {
1076 struct i915_request *w =
1077 container_of(p->waiter, typeof(*w), sched);
1078
1079 if (p->flags & I915_DEPENDENCY_WEAK)
1080 continue;
1081
1082
1083 if (w->engine != rq->engine)
1084 continue;
1085
1086
1087 GEM_BUG_ON(i915_request_has_initial_breadcrumb(w) &&
1088 __i915_request_has_started(w) &&
1089 !__i915_request_is_complete(rq));
1090
1091 if (!i915_request_is_ready(w))
1092 continue;
1093
1094 if (rq_prio(w) < rq_prio(rq))
1095 continue;
1096
1097 GEM_BUG_ON(rq_prio(w) > rq_prio(rq));
1098 GEM_BUG_ON(i915_request_is_active(w));
1099 list_move_tail(&w->sched.link, &list);
1100 }
1101
1102 rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
1103 } while (rq);
1104}
1105
1106static void defer_active(struct intel_engine_cs *engine)
1107{
1108 struct i915_request *rq;
1109
1110 rq = __unwind_incomplete_requests(engine);
1111 if (!rq)
1112 return;
1113
1114 defer_request(rq, i915_sched_lookup_priolist(engine->sched_engine,
1115 rq_prio(rq)));
1116}
1117
1118static bool
1119timeslice_yield(const struct intel_engine_execlists *el,
1120 const struct i915_request *rq)
1121{
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134 return rq->context->lrc.ccid == READ_ONCE(el->yield);
1135}
1136
1137static bool needs_timeslice(const struct intel_engine_cs *engine,
1138 const struct i915_request *rq)
1139{
1140 if (!intel_engine_has_timeslices(engine))
1141 return false;
1142
1143
1144 if (!rq || __i915_request_is_complete(rq))
1145 return false;
1146
1147
1148 if (READ_ONCE(engine->execlists.pending[0]))
1149 return false;
1150
1151
1152 if (!list_is_last_rcu(&rq->sched.link,
1153 &engine->sched_engine->requests)) {
1154 ENGINE_TRACE(engine, "timeslice required for second inflight context\n");
1155 return true;
1156 }
1157
1158
1159 if (!i915_sched_engine_is_empty(engine->sched_engine)) {
1160 ENGINE_TRACE(engine, "timeslice required for queue\n");
1161 return true;
1162 }
1163
1164 if (!RB_EMPTY_ROOT(&engine->execlists.virtual.rb_root)) {
1165 ENGINE_TRACE(engine, "timeslice required for virtual\n");
1166 return true;
1167 }
1168
1169 return false;
1170}
1171
1172static bool
1173timeslice_expired(struct intel_engine_cs *engine, const struct i915_request *rq)
1174{
1175 const struct intel_engine_execlists *el = &engine->execlists;
1176
1177 if (i915_request_has_nopreempt(rq) && __i915_request_has_started(rq))
1178 return false;
1179
1180 if (!needs_timeslice(engine, rq))
1181 return false;
1182
1183 return timer_expired(&el->timer) || timeslice_yield(el, rq);
1184}
1185
1186static unsigned long timeslice(const struct intel_engine_cs *engine)
1187{
1188 return READ_ONCE(engine->props.timeslice_duration_ms);
1189}
1190
1191static void start_timeslice(struct intel_engine_cs *engine)
1192{
1193 struct intel_engine_execlists *el = &engine->execlists;
1194 unsigned long duration;
1195
1196
1197 duration = 0;
1198 if (needs_timeslice(engine, *el->active)) {
1199
1200 if (timer_active(&el->timer)) {
1201
1202
1203
1204
1205
1206 if (!timer_pending(&el->timer))
1207 tasklet_hi_schedule(&engine->sched_engine->tasklet);
1208 return;
1209 }
1210
1211 duration = timeslice(engine);
1212 }
1213
1214 set_timer_ms(&el->timer, duration);
1215}
1216
1217static void record_preemption(struct intel_engine_execlists *execlists)
1218{
1219 (void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++);
1220}
1221
1222static unsigned long active_preempt_timeout(struct intel_engine_cs *engine,
1223 const struct i915_request *rq)
1224{
1225 if (!rq)
1226 return 0;
1227
1228
1229 if (unlikely(intel_context_is_banned(rq->context) || bad_request(rq)))
1230 return 1;
1231
1232 return READ_ONCE(engine->props.preempt_timeout_ms);
1233}
1234
1235static void set_preempt_timeout(struct intel_engine_cs *engine,
1236 const struct i915_request *rq)
1237{
1238 if (!intel_engine_has_preempt_reset(engine))
1239 return;
1240
1241 set_timer_ms(&engine->execlists.preempt,
1242 active_preempt_timeout(engine, rq));
1243}
1244
1245static bool completed(const struct i915_request *rq)
1246{
1247 if (i915_request_has_sentinel(rq))
1248 return false;
1249
1250 return __i915_request_is_complete(rq);
1251}
1252
1253static void execlists_dequeue(struct intel_engine_cs *engine)
1254{
1255 struct intel_engine_execlists * const execlists = &engine->execlists;
1256 struct i915_sched_engine * const sched_engine = engine->sched_engine;
1257 struct i915_request **port = execlists->pending;
1258 struct i915_request ** const last_port = port + execlists->port_mask;
1259 struct i915_request *last, * const *active;
1260 struct virtual_engine *ve;
1261 struct rb_node *rb;
1262 bool submit = false;
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286 spin_lock(&sched_engine->lock);
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297 active = execlists->active;
1298 while ((last = *active) && completed(last))
1299 active++;
1300
1301 if (last) {
1302 if (need_preempt(engine, last)) {
1303 ENGINE_TRACE(engine,
1304 "preempting last=%llx:%lld, prio=%d, hint=%d\n",
1305 last->fence.context,
1306 last->fence.seqno,
1307 last->sched.attr.priority,
1308 sched_engine->queue_priority_hint);
1309 record_preemption(execlists);
1310
1311
1312
1313
1314
1315
1316 ring_set_paused(engine, 1);
1317
1318
1319
1320
1321
1322
1323
1324
1325 __unwind_incomplete_requests(engine);
1326
1327 last = NULL;
1328 } else if (timeslice_expired(engine, last)) {
1329 ENGINE_TRACE(engine,
1330 "expired:%s last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n",
1331 yesno(timer_expired(&execlists->timer)),
1332 last->fence.context, last->fence.seqno,
1333 rq_prio(last),
1334 sched_engine->queue_priority_hint,
1335 yesno(timeslice_yield(execlists, last)));
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353 cancel_timer(&execlists->timer);
1354 ring_set_paused(engine, 1);
1355 defer_active(engine);
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370 last = NULL;
1371 } else {
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381 if (active[1]) {
1382
1383
1384
1385
1386 spin_unlock(&sched_engine->lock);
1387 return;
1388 }
1389 }
1390 }
1391
1392
1393 while ((ve = first_virtual_engine(engine))) {
1394 struct i915_request *rq;
1395
1396 spin_lock(&ve->base.sched_engine->lock);
1397
1398 rq = ve->request;
1399 if (unlikely(!virtual_matches(ve, rq, engine)))
1400 goto unlock;
1401
1402 GEM_BUG_ON(rq->engine != &ve->base);
1403 GEM_BUG_ON(rq->context != &ve->context);
1404
1405 if (unlikely(rq_prio(rq) < queue_prio(sched_engine))) {
1406 spin_unlock(&ve->base.sched_engine->lock);
1407 break;
1408 }
1409
1410 if (last && !can_merge_rq(last, rq)) {
1411 spin_unlock(&ve->base.sched_engine->lock);
1412 spin_unlock(&engine->sched_engine->lock);
1413 return;
1414 }
1415
1416 ENGINE_TRACE(engine,
1417 "virtual rq=%llx:%lld%s, new engine? %s\n",
1418 rq->fence.context,
1419 rq->fence.seqno,
1420 __i915_request_is_complete(rq) ? "!" :
1421 __i915_request_has_started(rq) ? "*" :
1422 "",
1423 yesno(engine != ve->siblings[0]));
1424
1425 WRITE_ONCE(ve->request, NULL);
1426 WRITE_ONCE(ve->base.sched_engine->queue_priority_hint, INT_MIN);
1427
1428 rb = &ve->nodes[engine->id].rb;
1429 rb_erase_cached(rb, &execlists->virtual);
1430 RB_CLEAR_NODE(rb);
1431
1432 GEM_BUG_ON(!(rq->execution_mask & engine->mask));
1433 WRITE_ONCE(rq->engine, engine);
1434
1435 if (__i915_request_submit(rq)) {
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449 virtual_xfer_context(ve, engine);
1450 GEM_BUG_ON(ve->siblings[0] != engine);
1451
1452 submit = true;
1453 last = rq;
1454 }
1455
1456 i915_request_put(rq);
1457unlock:
1458 spin_unlock(&ve->base.sched_engine->lock);
1459
1460
1461
1462
1463
1464
1465
1466
1467 if (submit)
1468 break;
1469 }
1470
1471 while ((rb = rb_first_cached(&sched_engine->queue))) {
1472 struct i915_priolist *p = to_priolist(rb);
1473 struct i915_request *rq, *rn;
1474
1475 priolist_for_each_request_consume(rq, rn, p) {
1476 bool merge = true;
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489 if (last && !can_merge_rq(last, rq)) {
1490
1491
1492
1493
1494
1495 if (port == last_port)
1496 goto done;
1497
1498
1499
1500
1501
1502
1503 if (last->context == rq->context)
1504 goto done;
1505
1506 if (i915_request_has_sentinel(last))
1507 goto done;
1508
1509
1510
1511
1512
1513
1514
1515 if (rq->execution_mask != engine->mask)
1516 goto done;
1517
1518
1519
1520
1521
1522
1523
1524
1525 if (ctx_single_port_submission(last->context) ||
1526 ctx_single_port_submission(rq->context))
1527 goto done;
1528
1529 merge = false;
1530 }
1531
1532 if (__i915_request_submit(rq)) {
1533 if (!merge) {
1534 *port++ = i915_request_get(last);
1535 last = NULL;
1536 }
1537
1538 GEM_BUG_ON(last &&
1539 !can_merge_ctx(last->context,
1540 rq->context));
1541 GEM_BUG_ON(last &&
1542 i915_seqno_passed(last->fence.seqno,
1543 rq->fence.seqno));
1544
1545 submit = true;
1546 last = rq;
1547 }
1548 }
1549
1550 rb_erase_cached(&p->node, &sched_engine->queue);
1551 i915_priolist_free(p);
1552 }
1553done:
1554 *port++ = i915_request_get(last);
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572 sched_engine->queue_priority_hint = queue_prio(sched_engine);
1573 i915_sched_engine_reset_on_empty(sched_engine);
1574 spin_unlock(&sched_engine->lock);
1575
1576
1577
1578
1579
1580
1581 if (submit &&
1582 memcmp(active,
1583 execlists->pending,
1584 (port - execlists->pending) * sizeof(*port))) {
1585 *port = NULL;
1586 while (port-- != execlists->pending)
1587 execlists_schedule_in(*port, port - execlists->pending);
1588
1589 WRITE_ONCE(execlists->yield, -1);
1590 set_preempt_timeout(engine, *active);
1591 execlists_submit_ports(engine);
1592 } else {
1593 ring_set_paused(engine, 0);
1594 while (port-- != execlists->pending)
1595 i915_request_put(*port);
1596 *execlists->pending = NULL;
1597 }
1598}
1599
1600static void execlists_dequeue_irq(struct intel_engine_cs *engine)
1601{
1602 local_irq_disable();
1603 execlists_dequeue(engine);
1604 local_irq_enable();
1605}
1606
1607static void clear_ports(struct i915_request **ports, int count)
1608{
1609 memset_p((void **)ports, NULL, count);
1610}
1611
1612static void
1613copy_ports(struct i915_request **dst, struct i915_request **src, int count)
1614{
1615
1616 while (count--)
1617 WRITE_ONCE(*dst++, *src++);
1618}
1619
1620static struct i915_request **
1621cancel_port_requests(struct intel_engine_execlists * const execlists,
1622 struct i915_request **inactive)
1623{
1624 struct i915_request * const *port;
1625
1626 for (port = execlists->pending; *port; port++)
1627 *inactive++ = *port;
1628 clear_ports(execlists->pending, ARRAY_SIZE(execlists->pending));
1629
1630
1631 for (port = xchg(&execlists->active, execlists->pending); *port; port++)
1632 *inactive++ = *port;
1633 clear_ports(execlists->inflight, ARRAY_SIZE(execlists->inflight));
1634
1635 smp_wmb();
1636 WRITE_ONCE(execlists->active, execlists->inflight);
1637
1638
1639 GEM_BUG_ON(execlists->pending[0]);
1640 cancel_timer(&execlists->timer);
1641 cancel_timer(&execlists->preempt);
1642
1643 return inactive;
1644}
1645
1646static void invalidate_csb_entries(const u64 *first, const u64 *last)
1647{
1648 clflush((void *)first);
1649 clflush((void *)last);
1650}
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691static inline bool
1692__gen12_csb_parse(bool ctx_to_valid, bool ctx_away_valid, bool new_queue,
1693 u8 switch_detail)
1694{
1695
1696
1697
1698
1699
1700
1701
1702 if (!ctx_away_valid || new_queue) {
1703 GEM_BUG_ON(!ctx_to_valid);
1704 return true;
1705 }
1706
1707
1708
1709
1710
1711
1712 GEM_BUG_ON(switch_detail);
1713 return false;
1714}
1715
1716static bool xehp_csb_parse(const u64 csb)
1717{
1718 return __gen12_csb_parse(XEHP_CSB_CTX_VALID(lower_32_bits(csb)),
1719 XEHP_CSB_CTX_VALID(upper_32_bits(csb)),
1720 upper_32_bits(csb) & XEHP_CTX_STATUS_SWITCHED_TO_NEW_QUEUE,
1721 GEN12_CTX_SWITCH_DETAIL(lower_32_bits(csb)));
1722}
1723
1724static bool gen12_csb_parse(const u64 csb)
1725{
1726 return __gen12_csb_parse(GEN12_CSB_CTX_VALID(lower_32_bits(csb)),
1727 GEN12_CSB_CTX_VALID(upper_32_bits(csb)),
1728 lower_32_bits(csb) & GEN12_CTX_STATUS_SWITCHED_TO_NEW_QUEUE,
1729 GEN12_CTX_SWITCH_DETAIL(upper_32_bits(csb)));
1730}
1731
1732static bool gen8_csb_parse(const u64 csb)
1733{
1734 return csb & (GEN8_CTX_STATUS_IDLE_ACTIVE | GEN8_CTX_STATUS_PREEMPTED);
1735}
1736
1737static noinline u64
1738wa_csb_read(const struct intel_engine_cs *engine, u64 * const csb)
1739{
1740 u64 entry;
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751 preempt_disable();
1752 if (wait_for_atomic_us((entry = READ_ONCE(*csb)) != -1, 10)) {
1753 int idx = csb - engine->execlists.csb_status;
1754 int status;
1755
1756 status = GEN8_EXECLISTS_STATUS_BUF;
1757 if (idx >= 6) {
1758 status = GEN11_EXECLISTS_STATUS_BUF2;
1759 idx -= 6;
1760 }
1761 status += sizeof(u64) * idx;
1762
1763 entry = intel_uncore_read64(engine->uncore,
1764 _MMIO(engine->mmio_base + status));
1765 }
1766 preempt_enable();
1767
1768 return entry;
1769}
1770
1771static u64 csb_read(const struct intel_engine_cs *engine, u64 * const csb)
1772{
1773 u64 entry = READ_ONCE(*csb);
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786 if (unlikely(entry == -1))
1787 entry = wa_csb_read(engine, csb);
1788
1789
1790 WRITE_ONCE(*csb, -1);
1791
1792
1793 return entry;
1794}
1795
1796static void new_timeslice(struct intel_engine_execlists *el)
1797{
1798
1799 cancel_timer(&el->timer);
1800}
1801
1802static struct i915_request **
1803process_csb(struct intel_engine_cs *engine, struct i915_request **inactive)
1804{
1805 struct intel_engine_execlists * const execlists = &engine->execlists;
1806 u64 * const buf = execlists->csb_status;
1807 const u8 num_entries = execlists->csb_size;
1808 struct i915_request **prev;
1809 u8 head, tail;
1810
1811
1812
1813
1814
1815
1816 GEM_BUG_ON(!tasklet_is_locked(&engine->sched_engine->tasklet) &&
1817 !reset_in_progress(engine));
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829 head = execlists->csb_head;
1830 tail = READ_ONCE(*execlists->csb_write);
1831 if (unlikely(head == tail))
1832 return inactive;
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850 execlists->csb_head = tail;
1851 ENGINE_TRACE(engine, "cs-irq head=%d, tail=%d\n", head, tail);
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861 rmb();
1862
1863
1864 prev = inactive;
1865 *prev = NULL;
1866
1867 do {
1868 bool promote;
1869 u64 csb;
1870
1871 if (++head == num_entries)
1872 head = 0;
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892 csb = csb_read(engine, buf + head);
1893 ENGINE_TRACE(engine, "csb[%d]: status=0x%08x:0x%08x\n",
1894 head, upper_32_bits(csb), lower_32_bits(csb));
1895
1896 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
1897 promote = xehp_csb_parse(csb);
1898 else if (GRAPHICS_VER(engine->i915) >= 12)
1899 promote = gen12_csb_parse(csb);
1900 else
1901 promote = gen8_csb_parse(csb);
1902 if (promote) {
1903 struct i915_request * const *old = execlists->active;
1904
1905 if (GEM_WARN_ON(!*execlists->pending)) {
1906 execlists->error_interrupt |= ERROR_CSB;
1907 break;
1908 }
1909
1910 ring_set_paused(engine, 0);
1911
1912
1913 WRITE_ONCE(execlists->active, execlists->pending);
1914 smp_wmb();
1915
1916
1917 trace_ports(execlists, "preempted", old);
1918 while (*old)
1919 *inactive++ = *old++;
1920
1921
1922 GEM_BUG_ON(!assert_pending_valid(execlists, "promote"));
1923 copy_ports(execlists->inflight,
1924 execlists->pending,
1925 execlists_num_ports(execlists));
1926 smp_wmb();
1927 WRITE_ONCE(execlists->active, execlists->inflight);
1928
1929
1930 ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
1931
1932 WRITE_ONCE(execlists->pending[0], NULL);
1933 } else {
1934 if (GEM_WARN_ON(!*execlists->active)) {
1935 execlists->error_interrupt |= ERROR_CSB;
1936 break;
1937 }
1938
1939
1940 trace_ports(execlists, "completed", execlists->active);
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952 if (GEM_SHOW_DEBUG() &&
1953 !__i915_request_is_complete(*execlists->active)) {
1954 struct i915_request *rq = *execlists->active;
1955 const u32 *regs __maybe_unused =
1956 rq->context->lrc_reg_state;
1957
1958 ENGINE_TRACE(engine,
1959 "context completed before request!\n");
1960 ENGINE_TRACE(engine,
1961 "ring:{start:0x%08x, head:%04x, tail:%04x, ctl:%08x, mode:%08x}\n",
1962 ENGINE_READ(engine, RING_START),
1963 ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR,
1964 ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR,
1965 ENGINE_READ(engine, RING_CTL),
1966 ENGINE_READ(engine, RING_MI_MODE));
1967 ENGINE_TRACE(engine,
1968 "rq:{start:%08x, head:%04x, tail:%04x, seqno:%llx:%d, hwsp:%d}, ",
1969 i915_ggtt_offset(rq->ring->vma),
1970 rq->head, rq->tail,
1971 rq->fence.context,
1972 lower_32_bits(rq->fence.seqno),
1973 hwsp_seqno(rq));
1974 ENGINE_TRACE(engine,
1975 "ctx:{start:%08x, head:%04x, tail:%04x}, ",
1976 regs[CTX_RING_START],
1977 regs[CTX_RING_HEAD],
1978 regs[CTX_RING_TAIL]);
1979 }
1980
1981 *inactive++ = *execlists->active++;
1982
1983 GEM_BUG_ON(execlists->active - execlists->inflight >
1984 execlists_num_ports(execlists));
1985 }
1986 } while (head != tail);
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999 invalidate_csb_entries(&buf[0], &buf[num_entries - 1]);
2000
2001
2002
2003
2004
2005
2006 if (*prev != *execlists->active)
2007 new_timeslice(execlists);
2008
2009 return inactive;
2010}
2011
2012static void post_process_csb(struct i915_request **port,
2013 struct i915_request **last)
2014{
2015 while (port != last)
2016 execlists_schedule_out(*port++);
2017}
2018
2019static void __execlists_hold(struct i915_request *rq)
2020{
2021 LIST_HEAD(list);
2022
2023 do {
2024 struct i915_dependency *p;
2025
2026 if (i915_request_is_active(rq))
2027 __i915_request_unsubmit(rq);
2028
2029 clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2030 list_move_tail(&rq->sched.link,
2031 &rq->engine->sched_engine->hold);
2032 i915_request_set_hold(rq);
2033 RQ_TRACE(rq, "on hold\n");
2034
2035 for_each_waiter(p, rq) {
2036 struct i915_request *w =
2037 container_of(p->waiter, typeof(*w), sched);
2038
2039 if (p->flags & I915_DEPENDENCY_WEAK)
2040 continue;
2041
2042
2043 if (w->engine != rq->engine)
2044 continue;
2045
2046 if (!i915_request_is_ready(w))
2047 continue;
2048
2049 if (__i915_request_is_complete(w))
2050 continue;
2051
2052 if (i915_request_on_hold(w))
2053 continue;
2054
2055 list_move_tail(&w->sched.link, &list);
2056 }
2057
2058 rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
2059 } while (rq);
2060}
2061
2062static bool execlists_hold(struct intel_engine_cs *engine,
2063 struct i915_request *rq)
2064{
2065 if (i915_request_on_hold(rq))
2066 return false;
2067
2068 spin_lock_irq(&engine->sched_engine->lock);
2069
2070 if (__i915_request_is_complete(rq)) {
2071 rq = NULL;
2072 goto unlock;
2073 }
2074
2075
2076
2077
2078
2079
2080
2081 GEM_BUG_ON(i915_request_on_hold(rq));
2082 GEM_BUG_ON(rq->engine != engine);
2083 __execlists_hold(rq);
2084 GEM_BUG_ON(list_empty(&engine->sched_engine->hold));
2085
2086unlock:
2087 spin_unlock_irq(&engine->sched_engine->lock);
2088 return rq;
2089}
2090
2091static bool hold_request(const struct i915_request *rq)
2092{
2093 struct i915_dependency *p;
2094 bool result = false;
2095
2096
2097
2098
2099
2100 rcu_read_lock();
2101 for_each_signaler(p, rq) {
2102 const struct i915_request *s =
2103 container_of(p->signaler, typeof(*s), sched);
2104
2105 if (s->engine != rq->engine)
2106 continue;
2107
2108 result = i915_request_on_hold(s);
2109 if (result)
2110 break;
2111 }
2112 rcu_read_unlock();
2113
2114 return result;
2115}
2116
2117static void __execlists_unhold(struct i915_request *rq)
2118{
2119 LIST_HEAD(list);
2120
2121 do {
2122 struct i915_dependency *p;
2123
2124 RQ_TRACE(rq, "hold release\n");
2125
2126 GEM_BUG_ON(!i915_request_on_hold(rq));
2127 GEM_BUG_ON(!i915_sw_fence_signaled(&rq->submit));
2128
2129 i915_request_clear_hold(rq);
2130 list_move_tail(&rq->sched.link,
2131 i915_sched_lookup_priolist(rq->engine->sched_engine,
2132 rq_prio(rq)));
2133 set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2134
2135
2136 for_each_waiter(p, rq) {
2137 struct i915_request *w =
2138 container_of(p->waiter, typeof(*w), sched);
2139
2140 if (p->flags & I915_DEPENDENCY_WEAK)
2141 continue;
2142
2143
2144 if (rq->fence.error)
2145 i915_request_set_error_once(w, rq->fence.error);
2146
2147 if (w->engine != rq->engine)
2148 continue;
2149
2150 if (!i915_request_on_hold(w))
2151 continue;
2152
2153
2154 if (hold_request(w))
2155 continue;
2156
2157 list_move_tail(&w->sched.link, &list);
2158 }
2159
2160 rq = list_first_entry_or_null(&list, typeof(*rq), sched.link);
2161 } while (rq);
2162}
2163
2164static void execlists_unhold(struct intel_engine_cs *engine,
2165 struct i915_request *rq)
2166{
2167 spin_lock_irq(&engine->sched_engine->lock);
2168
2169
2170
2171
2172
2173 __execlists_unhold(rq);
2174
2175 if (rq_prio(rq) > engine->sched_engine->queue_priority_hint) {
2176 engine->sched_engine->queue_priority_hint = rq_prio(rq);
2177 tasklet_hi_schedule(&engine->sched_engine->tasklet);
2178 }
2179
2180 spin_unlock_irq(&engine->sched_engine->lock);
2181}
2182
2183struct execlists_capture {
2184 struct work_struct work;
2185 struct i915_request *rq;
2186 struct i915_gpu_coredump *error;
2187};
2188
2189static void execlists_capture_work(struct work_struct *work)
2190{
2191 struct execlists_capture *cap = container_of(work, typeof(*cap), work);
2192 const gfp_t gfp = GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
2193 struct intel_engine_cs *engine = cap->rq->engine;
2194 struct intel_gt_coredump *gt = cap->error->gt;
2195 struct intel_engine_capture_vma *vma;
2196
2197
2198 vma = intel_engine_coredump_add_request(gt->engine, cap->rq, gfp);
2199 if (vma) {
2200 struct i915_vma_compress *compress =
2201 i915_vma_capture_prepare(gt);
2202
2203 intel_engine_coredump_add_vma(gt->engine, vma, compress);
2204 i915_vma_capture_finish(gt, compress);
2205 }
2206
2207 gt->simulated = gt->engine->simulated;
2208 cap->error->simulated = gt->simulated;
2209
2210
2211 i915_error_state_store(cap->error);
2212 i915_gpu_coredump_put(cap->error);
2213
2214
2215 execlists_unhold(engine, cap->rq);
2216 i915_request_put(cap->rq);
2217
2218 kfree(cap);
2219}
2220
2221static struct execlists_capture *capture_regs(struct intel_engine_cs *engine)
2222{
2223 const gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
2224 struct execlists_capture *cap;
2225
2226 cap = kmalloc(sizeof(*cap), gfp);
2227 if (!cap)
2228 return NULL;
2229
2230 cap->error = i915_gpu_coredump_alloc(engine->i915, gfp);
2231 if (!cap->error)
2232 goto err_cap;
2233
2234 cap->error->gt = intel_gt_coredump_alloc(engine->gt, gfp);
2235 if (!cap->error->gt)
2236 goto err_gpu;
2237
2238 cap->error->gt->engine = intel_engine_coredump_alloc(engine, gfp);
2239 if (!cap->error->gt->engine)
2240 goto err_gt;
2241
2242 cap->error->gt->engine->hung = true;
2243
2244 return cap;
2245
2246err_gt:
2247 kfree(cap->error->gt);
2248err_gpu:
2249 kfree(cap->error);
2250err_cap:
2251 kfree(cap);
2252 return NULL;
2253}
2254
2255static struct i915_request *
2256active_context(struct intel_engine_cs *engine, u32 ccid)
2257{
2258 const struct intel_engine_execlists * const el = &engine->execlists;
2259 struct i915_request * const *port, *rq;
2260
2261
2262
2263
2264
2265
2266
2267 for (port = el->active; (rq = *port); port++) {
2268 if (rq->context->lrc.ccid == ccid) {
2269 ENGINE_TRACE(engine,
2270 "ccid:%x found at active:%zd\n",
2271 ccid, port - el->active);
2272 return rq;
2273 }
2274 }
2275
2276 for (port = el->pending; (rq = *port); port++) {
2277 if (rq->context->lrc.ccid == ccid) {
2278 ENGINE_TRACE(engine,
2279 "ccid:%x found at pending:%zd\n",
2280 ccid, port - el->pending);
2281 return rq;
2282 }
2283 }
2284
2285 ENGINE_TRACE(engine, "ccid:%x not found\n", ccid);
2286 return NULL;
2287}
2288
2289static u32 active_ccid(struct intel_engine_cs *engine)
2290{
2291 return ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI);
2292}
2293
2294static void execlists_capture(struct intel_engine_cs *engine)
2295{
2296 struct execlists_capture *cap;
2297
2298 if (!IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR))
2299 return;
2300
2301
2302
2303
2304
2305
2306 cap = capture_regs(engine);
2307 if (!cap)
2308 return;
2309
2310 spin_lock_irq(&engine->sched_engine->lock);
2311 cap->rq = active_context(engine, active_ccid(engine));
2312 if (cap->rq) {
2313 cap->rq = active_request(cap->rq->context->timeline, cap->rq);
2314 cap->rq = i915_request_get_rcu(cap->rq);
2315 }
2316 spin_unlock_irq(&engine->sched_engine->lock);
2317 if (!cap->rq)
2318 goto err_free;
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340 if (!execlists_hold(engine, cap->rq))
2341 goto err_rq;
2342
2343 INIT_WORK(&cap->work, execlists_capture_work);
2344 schedule_work(&cap->work);
2345 return;
2346
2347err_rq:
2348 i915_request_put(cap->rq);
2349err_free:
2350 i915_gpu_coredump_put(cap->error);
2351 kfree(cap);
2352}
2353
2354static void execlists_reset(struct intel_engine_cs *engine, const char *msg)
2355{
2356 const unsigned int bit = I915_RESET_ENGINE + engine->id;
2357 unsigned long *lock = &engine->gt->reset.flags;
2358
2359 if (!intel_has_reset_engine(engine->gt))
2360 return;
2361
2362 if (test_and_set_bit(bit, lock))
2363 return;
2364
2365 ENGINE_TRACE(engine, "reset for %s\n", msg);
2366
2367
2368 tasklet_disable_nosync(&engine->sched_engine->tasklet);
2369
2370 ring_set_paused(engine, 1);
2371 execlists_capture(engine);
2372 intel_engine_reset(engine, msg);
2373
2374 tasklet_enable(&engine->sched_engine->tasklet);
2375 clear_and_wake_up_bit(bit, lock);
2376}
2377
2378static bool preempt_timeout(const struct intel_engine_cs *const engine)
2379{
2380 const struct timer_list *t = &engine->execlists.preempt;
2381
2382 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)
2383 return false;
2384
2385 if (!timer_expired(t))
2386 return false;
2387
2388 return engine->execlists.pending[0];
2389}
2390
2391
2392
2393
2394
2395static void execlists_submission_tasklet(struct tasklet_struct *t)
2396{
2397 struct i915_sched_engine *sched_engine =
2398 from_tasklet(sched_engine, t, tasklet);
2399 struct intel_engine_cs * const engine = sched_engine->private_data;
2400 struct i915_request *post[2 * EXECLIST_MAX_PORTS];
2401 struct i915_request **inactive;
2402
2403 rcu_read_lock();
2404 inactive = process_csb(engine, post);
2405 GEM_BUG_ON(inactive - post > ARRAY_SIZE(post));
2406
2407 if (unlikely(preempt_timeout(engine))) {
2408 cancel_timer(&engine->execlists.preempt);
2409 engine->execlists.error_interrupt |= ERROR_PREEMPT;
2410 }
2411
2412 if (unlikely(READ_ONCE(engine->execlists.error_interrupt))) {
2413 const char *msg;
2414
2415
2416 if (engine->execlists.error_interrupt & GENMASK(15, 0))
2417 msg = "CS error";
2418 else if (engine->execlists.error_interrupt & ERROR_CSB)
2419 msg = "invalid CSB event";
2420 else if (engine->execlists.error_interrupt & ERROR_PREEMPT)
2421 msg = "preemption time out";
2422 else
2423 msg = "internal error";
2424
2425 engine->execlists.error_interrupt = 0;
2426 execlists_reset(engine, msg);
2427 }
2428
2429 if (!engine->execlists.pending[0]) {
2430 execlists_dequeue_irq(engine);
2431 start_timeslice(engine);
2432 }
2433
2434 post_process_csb(post, inactive);
2435 rcu_read_unlock();
2436}
2437
2438static void execlists_irq_handler(struct intel_engine_cs *engine, u16 iir)
2439{
2440 bool tasklet = false;
2441
2442 if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) {
2443 u32 eir;
2444
2445
2446 eir = ENGINE_READ(engine, RING_EIR) & GENMASK(15, 0);
2447 ENGINE_TRACE(engine, "CS error: %x\n", eir);
2448
2449
2450 if (likely(eir)) {
2451 ENGINE_WRITE(engine, RING_EMR, ~0u);
2452 ENGINE_WRITE(engine, RING_EIR, eir);
2453 WRITE_ONCE(engine->execlists.error_interrupt, eir);
2454 tasklet = true;
2455 }
2456 }
2457
2458 if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) {
2459 WRITE_ONCE(engine->execlists.yield,
2460 ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI));
2461 ENGINE_TRACE(engine, "semaphore yield: %08x\n",
2462 engine->execlists.yield);
2463 if (del_timer(&engine->execlists.timer))
2464 tasklet = true;
2465 }
2466
2467 if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
2468 tasklet = true;
2469
2470 if (iir & GT_RENDER_USER_INTERRUPT)
2471 intel_engine_signal_breadcrumbs(engine);
2472
2473 if (tasklet)
2474 tasklet_hi_schedule(&engine->sched_engine->tasklet);
2475}
2476
2477static void __execlists_kick(struct intel_engine_execlists *execlists)
2478{
2479 struct intel_engine_cs *engine =
2480 container_of(execlists, typeof(*engine), execlists);
2481
2482
2483 tasklet_hi_schedule(&engine->sched_engine->tasklet);
2484}
2485
2486#define execlists_kick(t, member) \
2487 __execlists_kick(container_of(t, struct intel_engine_execlists, member))
2488
2489static void execlists_timeslice(struct timer_list *timer)
2490{
2491 execlists_kick(timer, timer);
2492}
2493
2494static void execlists_preempt(struct timer_list *timer)
2495{
2496 execlists_kick(timer, preempt);
2497}
2498
2499static void queue_request(struct intel_engine_cs *engine,
2500 struct i915_request *rq)
2501{
2502 GEM_BUG_ON(!list_empty(&rq->sched.link));
2503 list_add_tail(&rq->sched.link,
2504 i915_sched_lookup_priolist(engine->sched_engine,
2505 rq_prio(rq)));
2506 set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2507}
2508
2509static bool submit_queue(struct intel_engine_cs *engine,
2510 const struct i915_request *rq)
2511{
2512 struct i915_sched_engine *sched_engine = engine->sched_engine;
2513
2514 if (rq_prio(rq) <= sched_engine->queue_priority_hint)
2515 return false;
2516
2517 sched_engine->queue_priority_hint = rq_prio(rq);
2518 return true;
2519}
2520
2521static bool ancestor_on_hold(const struct intel_engine_cs *engine,
2522 const struct i915_request *rq)
2523{
2524 GEM_BUG_ON(i915_request_on_hold(rq));
2525 return !list_empty(&engine->sched_engine->hold) && hold_request(rq);
2526}
2527
2528static void execlists_submit_request(struct i915_request *request)
2529{
2530 struct intel_engine_cs *engine = request->engine;
2531 unsigned long flags;
2532
2533
2534 spin_lock_irqsave(&engine->sched_engine->lock, flags);
2535
2536 if (unlikely(ancestor_on_hold(engine, request))) {
2537 RQ_TRACE(request, "ancestor on hold\n");
2538 list_add_tail(&request->sched.link,
2539 &engine->sched_engine->hold);
2540 i915_request_set_hold(request);
2541 } else {
2542 queue_request(engine, request);
2543
2544 GEM_BUG_ON(i915_sched_engine_is_empty(engine->sched_engine));
2545 GEM_BUG_ON(list_empty(&request->sched.link));
2546
2547 if (submit_queue(engine, request))
2548 __execlists_kick(&engine->execlists);
2549 }
2550
2551 spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
2552}
2553
2554static int
2555__execlists_context_pre_pin(struct intel_context *ce,
2556 struct intel_engine_cs *engine,
2557 struct i915_gem_ww_ctx *ww, void **vaddr)
2558{
2559 int err;
2560
2561 err = lrc_pre_pin(ce, engine, ww, vaddr);
2562 if (err)
2563 return err;
2564
2565 if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) {
2566 lrc_init_state(ce, engine, *vaddr);
2567
2568 __i915_gem_object_flush_map(ce->state->obj, 0, engine->context_size);
2569 }
2570
2571 return 0;
2572}
2573
2574static int execlists_context_pre_pin(struct intel_context *ce,
2575 struct i915_gem_ww_ctx *ww,
2576 void **vaddr)
2577{
2578 return __execlists_context_pre_pin(ce, ce->engine, ww, vaddr);
2579}
2580
2581static int execlists_context_pin(struct intel_context *ce, void *vaddr)
2582{
2583 return lrc_pin(ce, ce->engine, vaddr);
2584}
2585
2586static int execlists_context_alloc(struct intel_context *ce)
2587{
2588 return lrc_alloc(ce, ce->engine);
2589}
2590
2591static void execlists_context_cancel_request(struct intel_context *ce,
2592 struct i915_request *rq)
2593{
2594 struct intel_engine_cs *engine = NULL;
2595
2596 i915_request_active_engine(rq, &engine);
2597
2598 if (engine && intel_engine_pulse(engine))
2599 intel_gt_handle_error(engine->gt, engine->mask, 0,
2600 "request cancellation by %s",
2601 current->comm);
2602}
2603
2604static const struct intel_context_ops execlists_context_ops = {
2605 .flags = COPS_HAS_INFLIGHT,
2606
2607 .alloc = execlists_context_alloc,
2608
2609 .cancel_request = execlists_context_cancel_request,
2610
2611 .pre_pin = execlists_context_pre_pin,
2612 .pin = execlists_context_pin,
2613 .unpin = lrc_unpin,
2614 .post_unpin = lrc_post_unpin,
2615
2616 .enter = intel_context_enter_engine,
2617 .exit = intel_context_exit_engine,
2618
2619 .reset = lrc_reset,
2620 .destroy = lrc_destroy,
2621
2622 .create_virtual = execlists_create_virtual,
2623};
2624
2625static int emit_pdps(struct i915_request *rq)
2626{
2627 const struct intel_engine_cs * const engine = rq->engine;
2628 struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(rq->context->vm);
2629 int err, i;
2630 u32 *cs;
2631
2632 GEM_BUG_ON(intel_vgpu_active(rq->engine->i915));
2633
2634
2635
2636
2637
2638
2639
2640
2641 cs = intel_ring_begin(rq, 2);
2642 if (IS_ERR(cs))
2643 return PTR_ERR(cs);
2644
2645 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
2646 *cs++ = MI_NOOP;
2647 intel_ring_advance(rq, cs);
2648
2649
2650 err = engine->emit_flush(rq, EMIT_FLUSH);
2651 if (err)
2652 return err;
2653
2654
2655 err = engine->emit_flush(rq, EMIT_INVALIDATE);
2656 if (err)
2657 return err;
2658
2659 cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
2660 if (IS_ERR(cs))
2661 return PTR_ERR(cs);
2662
2663
2664 *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
2665 for (i = GEN8_3LVL_PDPES; i--; ) {
2666 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
2667 u32 base = engine->mmio_base;
2668
2669 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
2670 *cs++ = upper_32_bits(pd_daddr);
2671 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
2672 *cs++ = lower_32_bits(pd_daddr);
2673 }
2674 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2675 intel_ring_advance(rq, cs);
2676
2677 intel_ring_advance(rq, cs);
2678
2679 return 0;
2680}
2681
2682static int execlists_request_alloc(struct i915_request *request)
2683{
2684 int ret;
2685
2686 GEM_BUG_ON(!intel_context_is_pinned(request->context));
2687
2688
2689
2690
2691
2692
2693 request->reserved_space += EXECLISTS_REQUEST_SIZE;
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703 if (!i915_vm_is_4lvl(request->context->vm)) {
2704 ret = emit_pdps(request);
2705 if (ret)
2706 return ret;
2707 }
2708
2709
2710 ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
2711 if (ret)
2712 return ret;
2713
2714 request->reserved_space -= EXECLISTS_REQUEST_SIZE;
2715 return 0;
2716}
2717
2718static void reset_csb_pointers(struct intel_engine_cs *engine)
2719{
2720 struct intel_engine_execlists * const execlists = &engine->execlists;
2721 const unsigned int reset_value = execlists->csb_size - 1;
2722
2723 ring_set_paused(engine, 0);
2724
2725
2726
2727
2728
2729 ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR,
2730 0xffff << 16 | reset_value << 8 | reset_value);
2731 ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742 execlists->csb_head = reset_value;
2743 WRITE_ONCE(*execlists->csb_write, reset_value);
2744 wmb();
2745
2746
2747 memset(execlists->csb_status, -1, (reset_value + 1) * sizeof(u64));
2748 invalidate_csb_entries(&execlists->csb_status[0],
2749 &execlists->csb_status[reset_value]);
2750
2751
2752 ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR,
2753 0xffff << 16 | reset_value << 8 | reset_value);
2754 ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR);
2755
2756 GEM_BUG_ON(READ_ONCE(*execlists->csb_write) != reset_value);
2757}
2758
2759static void sanitize_hwsp(struct intel_engine_cs *engine)
2760{
2761 struct intel_timeline *tl;
2762
2763 list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
2764 intel_timeline_reset_seqno(tl);
2765}
2766
2767static void execlists_sanitize(struct intel_engine_cs *engine)
2768{
2769 GEM_BUG_ON(execlists_active(&engine->execlists));
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
2781 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
2782
2783 reset_csb_pointers(engine);
2784
2785
2786
2787
2788
2789
2790 sanitize_hwsp(engine);
2791
2792
2793 clflush_cache_range(engine->status_page.addr, PAGE_SIZE);
2794}
2795
2796static void enable_error_interrupt(struct intel_engine_cs *engine)
2797{
2798 u32 status;
2799
2800 engine->execlists.error_interrupt = 0;
2801 ENGINE_WRITE(engine, RING_EMR, ~0u);
2802 ENGINE_WRITE(engine, RING_EIR, ~0u);
2803
2804 status = ENGINE_READ(engine, RING_ESR);
2805 if (unlikely(status)) {
2806 drm_err(&engine->i915->drm,
2807 "engine '%s' resumed still in error: %08x\n",
2808 engine->name, status);
2809 __intel_gt_reset(engine->gt, engine->mask);
2810 }
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836 ENGINE_WRITE(engine, RING_EMR, ~I915_ERROR_INSTRUCTION);
2837}
2838
2839static void enable_execlists(struct intel_engine_cs *engine)
2840{
2841 u32 mode;
2842
2843 assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
2844
2845 intel_engine_set_hwsp_writemask(engine, ~0u);
2846
2847 if (GRAPHICS_VER(engine->i915) >= 11)
2848 mode = _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE);
2849 else
2850 mode = _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE);
2851 ENGINE_WRITE_FW(engine, RING_MODE_GEN7, mode);
2852
2853 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
2854
2855 ENGINE_WRITE_FW(engine,
2856 RING_HWS_PGA,
2857 i915_ggtt_offset(engine->status_page.vma));
2858 ENGINE_POSTING_READ(engine, RING_HWS_PGA);
2859
2860 enable_error_interrupt(engine);
2861}
2862
2863static int execlists_resume(struct intel_engine_cs *engine)
2864{
2865 intel_mocs_init_engine(engine);
2866 intel_breadcrumbs_reset(engine->breadcrumbs);
2867
2868 enable_execlists(engine);
2869
2870 return 0;
2871}
2872
2873static void execlists_reset_prepare(struct intel_engine_cs *engine)
2874{
2875 ENGINE_TRACE(engine, "depth<-%d\n",
2876 atomic_read(&engine->sched_engine->tasklet.count));
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887 __tasklet_disable_sync_once(&engine->sched_engine->tasklet);
2888 GEM_BUG_ON(!reset_in_progress(engine));
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902 ring_set_paused(engine, 1);
2903 intel_engine_stop_cs(engine);
2904
2905 engine->execlists.reset_ccid = active_ccid(engine);
2906}
2907
2908static struct i915_request **
2909reset_csb(struct intel_engine_cs *engine, struct i915_request **inactive)
2910{
2911 struct intel_engine_execlists * const execlists = &engine->execlists;
2912
2913 mb();
2914 clflush(execlists->csb_write);
2915 mb();
2916
2917 inactive = process_csb(engine, inactive);
2918
2919
2920 reset_csb_pointers(engine);
2921
2922 return inactive;
2923}
2924
2925static void
2926execlists_reset_active(struct intel_engine_cs *engine, bool stalled)
2927{
2928 struct intel_context *ce;
2929 struct i915_request *rq;
2930 u32 head;
2931
2932
2933
2934
2935
2936
2937 rq = active_context(engine, engine->execlists.reset_ccid);
2938 if (!rq)
2939 return;
2940
2941 ce = rq->context;
2942 GEM_BUG_ON(!i915_vma_is_pinned(ce->state));
2943
2944 if (__i915_request_is_complete(rq)) {
2945
2946 head = intel_ring_wrap(ce->ring, rq->tail);
2947 goto out_replay;
2948 }
2949
2950
2951 GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
2952
2953
2954 GEM_BUG_ON(i915_active_is_idle(&ce->active));
2955
2956 rq = active_request(ce->timeline, rq);
2957 head = intel_ring_wrap(ce->ring, rq->head);
2958 GEM_BUG_ON(head == ce->ring->tail);
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972 if (!__i915_request_has_started(rq))
2973 goto out_replay;
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986 __i915_request_reset(rq, stalled);
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996out_replay:
2997 ENGINE_TRACE(engine, "replay {head:%04x, tail:%04x}\n",
2998 head, ce->ring->tail);
2999 lrc_reset_regs(ce, engine);
3000 ce->lrc.lrca = lrc_update_regs(ce, engine, head);
3001}
3002
3003static void execlists_reset_csb(struct intel_engine_cs *engine, bool stalled)
3004{
3005 struct intel_engine_execlists * const execlists = &engine->execlists;
3006 struct i915_request *post[2 * EXECLIST_MAX_PORTS];
3007 struct i915_request **inactive;
3008
3009 rcu_read_lock();
3010 inactive = reset_csb(engine, post);
3011
3012 execlists_reset_active(engine, true);
3013
3014 inactive = cancel_port_requests(execlists, inactive);
3015 post_process_csb(post, inactive);
3016 rcu_read_unlock();
3017}
3018
3019static void execlists_reset_rewind(struct intel_engine_cs *engine, bool stalled)
3020{
3021 unsigned long flags;
3022
3023 ENGINE_TRACE(engine, "\n");
3024
3025
3026 execlists_reset_csb(engine, stalled);
3027
3028
3029 rcu_read_lock();
3030 spin_lock_irqsave(&engine->sched_engine->lock, flags);
3031 __unwind_incomplete_requests(engine);
3032 spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
3033 rcu_read_unlock();
3034}
3035
3036static void nop_submission_tasklet(struct tasklet_struct *t)
3037{
3038 struct i915_sched_engine *sched_engine =
3039 from_tasklet(sched_engine, t, tasklet);
3040 struct intel_engine_cs * const engine = sched_engine->private_data;
3041
3042
3043 WRITE_ONCE(engine->sched_engine->queue_priority_hint, INT_MIN);
3044}
3045
3046static void execlists_reset_cancel(struct intel_engine_cs *engine)
3047{
3048 struct intel_engine_execlists * const execlists = &engine->execlists;
3049 struct i915_sched_engine * const sched_engine = engine->sched_engine;
3050 struct i915_request *rq, *rn;
3051 struct rb_node *rb;
3052 unsigned long flags;
3053
3054 ENGINE_TRACE(engine, "\n");
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070 execlists_reset_csb(engine, true);
3071
3072 rcu_read_lock();
3073 spin_lock_irqsave(&engine->sched_engine->lock, flags);
3074
3075
3076 list_for_each_entry(rq, &engine->sched_engine->requests, sched.link)
3077 i915_request_put(i915_request_mark_eio(rq));
3078 intel_engine_signal_breadcrumbs(engine);
3079
3080
3081 while ((rb = rb_first_cached(&sched_engine->queue))) {
3082 struct i915_priolist *p = to_priolist(rb);
3083
3084 priolist_for_each_request_consume(rq, rn, p) {
3085 if (i915_request_mark_eio(rq)) {
3086 __i915_request_submit(rq);
3087 i915_request_put(rq);
3088 }
3089 }
3090
3091 rb_erase_cached(&p->node, &sched_engine->queue);
3092 i915_priolist_free(p);
3093 }
3094
3095
3096 list_for_each_entry(rq, &sched_engine->hold, sched.link)
3097 i915_request_put(i915_request_mark_eio(rq));
3098
3099
3100 while ((rb = rb_first_cached(&execlists->virtual))) {
3101 struct virtual_engine *ve =
3102 rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
3103
3104 rb_erase_cached(rb, &execlists->virtual);
3105 RB_CLEAR_NODE(rb);
3106
3107 spin_lock(&ve->base.sched_engine->lock);
3108 rq = fetch_and_zero(&ve->request);
3109 if (rq) {
3110 if (i915_request_mark_eio(rq)) {
3111 rq->engine = engine;
3112 __i915_request_submit(rq);
3113 i915_request_put(rq);
3114 }
3115 i915_request_put(rq);
3116
3117 ve->base.sched_engine->queue_priority_hint = INT_MIN;
3118 }
3119 spin_unlock(&ve->base.sched_engine->lock);
3120 }
3121
3122
3123
3124 sched_engine->queue_priority_hint = INT_MIN;
3125 sched_engine->queue = RB_ROOT_CACHED;
3126
3127 GEM_BUG_ON(__tasklet_is_enabled(&engine->sched_engine->tasklet));
3128 engine->sched_engine->tasklet.callback = nop_submission_tasklet;
3129
3130 spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
3131 rcu_read_unlock();
3132}
3133
3134static void execlists_reset_finish(struct intel_engine_cs *engine)
3135{
3136 struct intel_engine_execlists * const execlists = &engine->execlists;
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148 GEM_BUG_ON(!reset_in_progress(engine));
3149
3150
3151 if (__tasklet_enable(&engine->sched_engine->tasklet))
3152 __execlists_kick(execlists);
3153
3154 ENGINE_TRACE(engine, "depth->%d\n",
3155 atomic_read(&engine->sched_engine->tasklet.count));
3156}
3157
3158static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
3159{
3160 ENGINE_WRITE(engine, RING_IMR,
3161 ~(engine->irq_enable_mask | engine->irq_keep_mask));
3162 ENGINE_POSTING_READ(engine, RING_IMR);
3163}
3164
3165static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
3166{
3167 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
3168}
3169
3170static void execlists_park(struct intel_engine_cs *engine)
3171{
3172 cancel_timer(&engine->execlists.timer);
3173 cancel_timer(&engine->execlists.preempt);
3174}
3175
3176static void add_to_engine(struct i915_request *rq)
3177{
3178 lockdep_assert_held(&rq->engine->sched_engine->lock);
3179 list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests);
3180}
3181
3182static void remove_from_engine(struct i915_request *rq)
3183{
3184 struct intel_engine_cs *engine, *locked;
3185
3186
3187
3188
3189
3190
3191
3192 locked = READ_ONCE(rq->engine);
3193 spin_lock_irq(&locked->sched_engine->lock);
3194 while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
3195 spin_unlock(&locked->sched_engine->lock);
3196 spin_lock(&engine->sched_engine->lock);
3197 locked = engine;
3198 }
3199 list_del_init(&rq->sched.link);
3200
3201 clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
3202 clear_bit(I915_FENCE_FLAG_HOLD, &rq->fence.flags);
3203
3204
3205 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
3206
3207 spin_unlock_irq(&locked->sched_engine->lock);
3208
3209 i915_request_notify_execute_cb_imm(rq);
3210}
3211
3212static bool can_preempt(struct intel_engine_cs *engine)
3213{
3214 if (GRAPHICS_VER(engine->i915) > 8)
3215 return true;
3216
3217
3218 return engine->class != RENDER_CLASS;
3219}
3220
3221static void kick_execlists(const struct i915_request *rq, int prio)
3222{
3223 struct intel_engine_cs *engine = rq->engine;
3224 struct i915_sched_engine *sched_engine = engine->sched_engine;
3225 const struct i915_request *inflight;
3226
3227
3228
3229
3230
3231 if (prio <= sched_engine->queue_priority_hint)
3232 return;
3233
3234 rcu_read_lock();
3235
3236
3237 inflight = execlists_active(&engine->execlists);
3238 if (!inflight)
3239 goto unlock;
3240
3241
3242
3243
3244
3245 if (inflight->context == rq->context)
3246 goto unlock;
3247
3248 ENGINE_TRACE(engine,
3249 "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
3250 prio,
3251 rq->fence.context, rq->fence.seqno,
3252 inflight->fence.context, inflight->fence.seqno,
3253 inflight->sched.attr.priority);
3254
3255 sched_engine->queue_priority_hint = prio;
3256
3257
3258
3259
3260
3261
3262
3263
3264 if (prio >= max(I915_PRIORITY_NORMAL, rq_prio(inflight)))
3265 tasklet_hi_schedule(&sched_engine->tasklet);
3266
3267unlock:
3268 rcu_read_unlock();
3269}
3270
3271static void execlists_set_default_submission(struct intel_engine_cs *engine)
3272{
3273 engine->submit_request = execlists_submit_request;
3274 engine->sched_engine->schedule = i915_schedule;
3275 engine->sched_engine->kick_backend = kick_execlists;
3276 engine->sched_engine->tasklet.callback = execlists_submission_tasklet;
3277}
3278
3279static void execlists_shutdown(struct intel_engine_cs *engine)
3280{
3281
3282 del_timer_sync(&engine->execlists.timer);
3283 del_timer_sync(&engine->execlists.preempt);
3284 tasklet_kill(&engine->sched_engine->tasklet);
3285}
3286
3287static void execlists_release(struct intel_engine_cs *engine)
3288{
3289 engine->sanitize = NULL;
3290
3291 execlists_shutdown(engine);
3292
3293 intel_engine_cleanup_common(engine);
3294 lrc_fini_wa_ctx(engine);
3295}
3296
3297static void
3298logical_ring_default_vfuncs(struct intel_engine_cs *engine)
3299{
3300
3301
3302 engine->resume = execlists_resume;
3303
3304 engine->cops = &execlists_context_ops;
3305 engine->request_alloc = execlists_request_alloc;
3306 engine->add_active_request = add_to_engine;
3307 engine->remove_active_request = remove_from_engine;
3308
3309 engine->reset.prepare = execlists_reset_prepare;
3310 engine->reset.rewind = execlists_reset_rewind;
3311 engine->reset.cancel = execlists_reset_cancel;
3312 engine->reset.finish = execlists_reset_finish;
3313
3314 engine->park = execlists_park;
3315 engine->unpark = NULL;
3316
3317 engine->emit_flush = gen8_emit_flush_xcs;
3318 engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
3319 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;
3320 if (GRAPHICS_VER(engine->i915) >= 12) {
3321 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;
3322 engine->emit_flush = gen12_emit_flush_xcs;
3323 }
3324 engine->set_default_submission = execlists_set_default_submission;
3325
3326 if (GRAPHICS_VER(engine->i915) < 11) {
3327 engine->irq_enable = gen8_logical_ring_enable_irq;
3328 engine->irq_disable = gen8_logical_ring_disable_irq;
3329 } else {
3330
3331
3332
3333
3334
3335
3336 }
3337 intel_engine_set_irq_handler(engine, execlists_irq_handler);
3338
3339 engine->flags |= I915_ENGINE_SUPPORTS_STATS;
3340 if (!intel_vgpu_active(engine->i915)) {
3341 engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
3342 if (can_preempt(engine)) {
3343 engine->flags |= I915_ENGINE_HAS_PREEMPTION;
3344 if (IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
3345 engine->flags |= I915_ENGINE_HAS_TIMESLICES;
3346 }
3347 }
3348
3349 if (intel_engine_has_preemption(engine))
3350 engine->emit_bb_start = gen8_emit_bb_start;
3351 else
3352 engine->emit_bb_start = gen8_emit_bb_start_noarb;
3353}
3354
3355static void logical_ring_default_irqs(struct intel_engine_cs *engine)
3356{
3357 unsigned int shift = 0;
3358
3359 if (GRAPHICS_VER(engine->i915) < 11) {
3360 const u8 irq_shifts[] = {
3361 [RCS0] = GEN8_RCS_IRQ_SHIFT,
3362 [BCS0] = GEN8_BCS_IRQ_SHIFT,
3363 [VCS0] = GEN8_VCS0_IRQ_SHIFT,
3364 [VCS1] = GEN8_VCS1_IRQ_SHIFT,
3365 [VECS0] = GEN8_VECS_IRQ_SHIFT,
3366 };
3367
3368 shift = irq_shifts[engine->id];
3369 }
3370
3371 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
3372 engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
3373 engine->irq_keep_mask |= GT_CS_MASTER_ERROR_INTERRUPT << shift;
3374 engine->irq_keep_mask |= GT_WAIT_SEMAPHORE_INTERRUPT << shift;
3375}
3376
3377static void rcs_submission_override(struct intel_engine_cs *engine)
3378{
3379 switch (GRAPHICS_VER(engine->i915)) {
3380 case 12:
3381 engine->emit_flush = gen12_emit_flush_rcs;
3382 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;
3383 break;
3384 case 11:
3385 engine->emit_flush = gen11_emit_flush_rcs;
3386 engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;
3387 break;
3388 default:
3389 engine->emit_flush = gen8_emit_flush_rcs;
3390 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
3391 break;
3392 }
3393}
3394
3395int intel_execlists_submission_setup(struct intel_engine_cs *engine)
3396{
3397 struct intel_engine_execlists * const execlists = &engine->execlists;
3398 struct drm_i915_private *i915 = engine->i915;
3399 struct intel_uncore *uncore = engine->uncore;
3400 u32 base = engine->mmio_base;
3401
3402 tasklet_setup(&engine->sched_engine->tasklet, execlists_submission_tasklet);
3403 timer_setup(&engine->execlists.timer, execlists_timeslice, 0);
3404 timer_setup(&engine->execlists.preempt, execlists_preempt, 0);
3405
3406 logical_ring_default_vfuncs(engine);
3407 logical_ring_default_irqs(engine);
3408
3409 if (engine->class == RENDER_CLASS)
3410 rcs_submission_override(engine);
3411
3412 lrc_init_wa_ctx(engine);
3413
3414 if (HAS_LOGICAL_RING_ELSQ(i915)) {
3415 execlists->submit_reg = uncore->regs +
3416 i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(base));
3417 execlists->ctrl_reg = uncore->regs +
3418 i915_mmio_reg_offset(RING_EXECLIST_CONTROL(base));
3419
3420 engine->fw_domain = intel_uncore_forcewake_for_reg(engine->uncore,
3421 RING_EXECLIST_CONTROL(engine->mmio_base),
3422 FW_REG_WRITE);
3423 } else {
3424 execlists->submit_reg = uncore->regs +
3425 i915_mmio_reg_offset(RING_ELSP(base));
3426 }
3427
3428 execlists->csb_status =
3429 (u64 *)&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
3430
3431 execlists->csb_write =
3432 &engine->status_page.addr[intel_hws_csb_write_index(i915)];
3433
3434 if (GRAPHICS_VER(i915) < 11)
3435 execlists->csb_size = GEN8_CSB_ENTRIES;
3436 else
3437 execlists->csb_size = GEN11_CSB_ENTRIES;
3438
3439 engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0);
3440 if (GRAPHICS_VER(engine->i915) >= 11 &&
3441 GRAPHICS_VER_FULL(engine->i915) < IP_VER(12, 50)) {
3442 execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32);
3443 execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32);
3444 }
3445
3446
3447 engine->sanitize = execlists_sanitize;
3448 engine->release = execlists_release;
3449
3450 return 0;
3451}
3452
3453static struct list_head *virtual_queue(struct virtual_engine *ve)
3454{
3455 return &ve->base.sched_engine->default_priolist.requests;
3456}
3457
3458static void rcu_virtual_context_destroy(struct work_struct *wrk)
3459{
3460 struct virtual_engine *ve =
3461 container_of(wrk, typeof(*ve), rcu.work);
3462 unsigned int n;
3463
3464 GEM_BUG_ON(ve->context.inflight);
3465
3466
3467 if (unlikely(ve->request)) {
3468 struct i915_request *old;
3469
3470 spin_lock_irq(&ve->base.sched_engine->lock);
3471
3472 old = fetch_and_zero(&ve->request);
3473 if (old) {
3474 GEM_BUG_ON(!__i915_request_is_complete(old));
3475 __i915_request_submit(old);
3476 i915_request_put(old);
3477 }
3478
3479 spin_unlock_irq(&ve->base.sched_engine->lock);
3480 }
3481
3482
3483
3484
3485
3486
3487
3488
3489 tasklet_kill(&ve->base.sched_engine->tasklet);
3490
3491
3492 for (n = 0; n < ve->num_siblings; n++) {
3493 struct intel_engine_cs *sibling = ve->siblings[n];
3494 struct rb_node *node = &ve->nodes[sibling->id].rb;
3495
3496 if (RB_EMPTY_NODE(node))
3497 continue;
3498
3499 spin_lock_irq(&sibling->sched_engine->lock);
3500
3501
3502 if (!RB_EMPTY_NODE(node))
3503 rb_erase_cached(node, &sibling->execlists.virtual);
3504
3505 spin_unlock_irq(&sibling->sched_engine->lock);
3506 }
3507 GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.sched_engine->tasklet));
3508 GEM_BUG_ON(!list_empty(virtual_queue(ve)));
3509
3510 lrc_fini(&ve->context);
3511 intel_context_fini(&ve->context);
3512
3513 if (ve->base.breadcrumbs)
3514 intel_breadcrumbs_put(ve->base.breadcrumbs);
3515 if (ve->base.sched_engine)
3516 i915_sched_engine_put(ve->base.sched_engine);
3517 intel_engine_free_request_pool(&ve->base);
3518
3519 kfree(ve);
3520}
3521
3522static void virtual_context_destroy(struct kref *kref)
3523{
3524 struct virtual_engine *ve =
3525 container_of(kref, typeof(*ve), context.ref);
3526
3527 GEM_BUG_ON(!list_empty(&ve->context.signals));
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539 INIT_RCU_WORK(&ve->rcu, rcu_virtual_context_destroy);
3540 queue_rcu_work(system_wq, &ve->rcu);
3541}
3542
3543static void virtual_engine_initial_hint(struct virtual_engine *ve)
3544{
3545 int swp;
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560 swp = prandom_u32_max(ve->num_siblings);
3561 if (swp)
3562 swap(ve->siblings[swp], ve->siblings[0]);
3563}
3564
3565static int virtual_context_alloc(struct intel_context *ce)
3566{
3567 struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3568
3569 return lrc_alloc(ce, ve->siblings[0]);
3570}
3571
3572static int virtual_context_pre_pin(struct intel_context *ce,
3573 struct i915_gem_ww_ctx *ww,
3574 void **vaddr)
3575{
3576 struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3577
3578
3579 return __execlists_context_pre_pin(ce, ve->siblings[0], ww, vaddr);
3580}
3581
3582static int virtual_context_pin(struct intel_context *ce, void *vaddr)
3583{
3584 struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3585
3586 return lrc_pin(ce, ve->siblings[0], vaddr);
3587}
3588
3589static void virtual_context_enter(struct intel_context *ce)
3590{
3591 struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3592 unsigned int n;
3593
3594 for (n = 0; n < ve->num_siblings; n++)
3595 intel_engine_pm_get(ve->siblings[n]);
3596
3597 intel_timeline_enter(ce->timeline);
3598}
3599
3600static void virtual_context_exit(struct intel_context *ce)
3601{
3602 struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
3603 unsigned int n;
3604
3605 intel_timeline_exit(ce->timeline);
3606
3607 for (n = 0; n < ve->num_siblings; n++)
3608 intel_engine_pm_put(ve->siblings[n]);
3609}
3610
3611static struct intel_engine_cs *
3612virtual_get_sibling(struct intel_engine_cs *engine, unsigned int sibling)
3613{
3614 struct virtual_engine *ve = to_virtual_engine(engine);
3615
3616 if (sibling >= ve->num_siblings)
3617 return NULL;
3618
3619 return ve->siblings[sibling];
3620}
3621
3622static const struct intel_context_ops virtual_context_ops = {
3623 .flags = COPS_HAS_INFLIGHT,
3624
3625 .alloc = virtual_context_alloc,
3626
3627 .cancel_request = execlists_context_cancel_request,
3628
3629 .pre_pin = virtual_context_pre_pin,
3630 .pin = virtual_context_pin,
3631 .unpin = lrc_unpin,
3632 .post_unpin = lrc_post_unpin,
3633
3634 .enter = virtual_context_enter,
3635 .exit = virtual_context_exit,
3636
3637 .destroy = virtual_context_destroy,
3638
3639 .get_sibling = virtual_get_sibling,
3640};
3641
3642static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve)
3643{
3644 struct i915_request *rq;
3645 intel_engine_mask_t mask;
3646
3647 rq = READ_ONCE(ve->request);
3648 if (!rq)
3649 return 0;
3650
3651
3652 mask = rq->execution_mask;
3653 if (unlikely(!mask)) {
3654
3655 i915_request_set_error_once(rq, -ENODEV);
3656 mask = ve->siblings[0]->mask;
3657 }
3658
3659 ENGINE_TRACE(&ve->base, "rq=%llx:%lld, mask=%x, prio=%d\n",
3660 rq->fence.context, rq->fence.seqno,
3661 mask, ve->base.sched_engine->queue_priority_hint);
3662
3663 return mask;
3664}
3665
3666static void virtual_submission_tasklet(struct tasklet_struct *t)
3667{
3668 struct i915_sched_engine *sched_engine =
3669 from_tasklet(sched_engine, t, tasklet);
3670 struct virtual_engine * const ve =
3671 (struct virtual_engine *)sched_engine->private_data;
3672 const int prio = READ_ONCE(sched_engine->queue_priority_hint);
3673 intel_engine_mask_t mask;
3674 unsigned int n;
3675
3676 rcu_read_lock();
3677 mask = virtual_submission_mask(ve);
3678 rcu_read_unlock();
3679 if (unlikely(!mask))
3680 return;
3681
3682 for (n = 0; n < ve->num_siblings; n++) {
3683 struct intel_engine_cs *sibling = READ_ONCE(ve->siblings[n]);
3684 struct ve_node * const node = &ve->nodes[sibling->id];
3685 struct rb_node **parent, *rb;
3686 bool first;
3687
3688 if (!READ_ONCE(ve->request))
3689 break;
3690
3691 spin_lock_irq(&sibling->sched_engine->lock);
3692
3693 if (unlikely(!(mask & sibling->mask))) {
3694 if (!RB_EMPTY_NODE(&node->rb)) {
3695 rb_erase_cached(&node->rb,
3696 &sibling->execlists.virtual);
3697 RB_CLEAR_NODE(&node->rb);
3698 }
3699
3700 goto unlock_engine;
3701 }
3702
3703 if (unlikely(!RB_EMPTY_NODE(&node->rb))) {
3704
3705
3706
3707
3708 first = rb_first_cached(&sibling->execlists.virtual) ==
3709 &node->rb;
3710 if (prio == node->prio || (prio > node->prio && first))
3711 goto submit_engine;
3712
3713 rb_erase_cached(&node->rb, &sibling->execlists.virtual);
3714 }
3715
3716 rb = NULL;
3717 first = true;
3718 parent = &sibling->execlists.virtual.rb_root.rb_node;
3719 while (*parent) {
3720 struct ve_node *other;
3721
3722 rb = *parent;
3723 other = rb_entry(rb, typeof(*other), rb);
3724 if (prio > other->prio) {
3725 parent = &rb->rb_left;
3726 } else {
3727 parent = &rb->rb_right;
3728 first = false;
3729 }
3730 }
3731
3732 rb_link_node(&node->rb, rb, parent);
3733 rb_insert_color_cached(&node->rb,
3734 &sibling->execlists.virtual,
3735 first);
3736
3737submit_engine:
3738 GEM_BUG_ON(RB_EMPTY_NODE(&node->rb));
3739 node->prio = prio;
3740 if (first && prio > sibling->sched_engine->queue_priority_hint)
3741 tasklet_hi_schedule(&sibling->sched_engine->tasklet);
3742
3743unlock_engine:
3744 spin_unlock_irq(&sibling->sched_engine->lock);
3745
3746 if (intel_context_inflight(&ve->context))
3747 break;
3748 }
3749}
3750
3751static void virtual_submit_request(struct i915_request *rq)
3752{
3753 struct virtual_engine *ve = to_virtual_engine(rq->engine);
3754 unsigned long flags;
3755
3756 ENGINE_TRACE(&ve->base, "rq=%llx:%lld\n",
3757 rq->fence.context,
3758 rq->fence.seqno);
3759
3760 GEM_BUG_ON(ve->base.submit_request != virtual_submit_request);
3761
3762 spin_lock_irqsave(&ve->base.sched_engine->lock, flags);
3763
3764
3765 if (__i915_request_is_complete(rq)) {
3766 __i915_request_submit(rq);
3767 goto unlock;
3768 }
3769
3770 if (ve->request) {
3771 GEM_BUG_ON(!__i915_request_is_complete(ve->request));
3772 __i915_request_submit(ve->request);
3773 i915_request_put(ve->request);
3774 }
3775
3776 ve->base.sched_engine->queue_priority_hint = rq_prio(rq);
3777 ve->request = i915_request_get(rq);
3778
3779 GEM_BUG_ON(!list_empty(virtual_queue(ve)));
3780 list_move_tail(&rq->sched.link, virtual_queue(ve));
3781
3782 tasklet_hi_schedule(&ve->base.sched_engine->tasklet);
3783
3784unlock:
3785 spin_unlock_irqrestore(&ve->base.sched_engine->lock, flags);
3786}
3787
3788static struct intel_context *
3789execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count)
3790{
3791 struct virtual_engine *ve;
3792 unsigned int n;
3793 int err;
3794
3795 ve = kzalloc(struct_size(ve, siblings, count), GFP_KERNEL);
3796 if (!ve)
3797 return ERR_PTR(-ENOMEM);
3798
3799 ve->base.i915 = siblings[0]->i915;
3800 ve->base.gt = siblings[0]->gt;
3801 ve->base.uncore = siblings[0]->uncore;
3802 ve->base.id = -1;
3803
3804 ve->base.class = OTHER_CLASS;
3805 ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
3806 ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
3807 ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822 ve->base.saturated = ALL_ENGINES;
3823
3824 snprintf(ve->base.name, sizeof(ve->base.name), "virtual");
3825
3826 intel_engine_init_execlists(&ve->base);
3827
3828 ve->base.sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);
3829 if (!ve->base.sched_engine) {
3830 err = -ENOMEM;
3831 goto err_put;
3832 }
3833 ve->base.sched_engine->private_data = &ve->base;
3834
3835 ve->base.cops = &virtual_context_ops;
3836 ve->base.request_alloc = execlists_request_alloc;
3837
3838 ve->base.sched_engine->schedule = i915_schedule;
3839 ve->base.sched_engine->kick_backend = kick_execlists;
3840 ve->base.submit_request = virtual_submit_request;
3841
3842 INIT_LIST_HEAD(virtual_queue(ve));
3843 tasklet_setup(&ve->base.sched_engine->tasklet, virtual_submission_tasklet);
3844
3845 intel_context_init(&ve->context, &ve->base);
3846
3847 ve->base.breadcrumbs = intel_breadcrumbs_create(NULL);
3848 if (!ve->base.breadcrumbs) {
3849 err = -ENOMEM;
3850 goto err_put;
3851 }
3852
3853 for (n = 0; n < count; n++) {
3854 struct intel_engine_cs *sibling = siblings[n];
3855
3856 GEM_BUG_ON(!is_power_of_2(sibling->mask));
3857 if (sibling->mask & ve->base.mask) {
3858 DRM_DEBUG("duplicate %s entry in load balancer\n",
3859 sibling->name);
3860 err = -EINVAL;
3861 goto err_put;
3862 }
3863
3864
3865
3866
3867
3868
3869
3870
3871 if (sibling->sched_engine->tasklet.callback !=
3872 execlists_submission_tasklet) {
3873 err = -ENODEV;
3874 goto err_put;
3875 }
3876
3877 GEM_BUG_ON(RB_EMPTY_NODE(&ve->nodes[sibling->id].rb));
3878 RB_CLEAR_NODE(&ve->nodes[sibling->id].rb);
3879
3880 ve->siblings[ve->num_siblings++] = sibling;
3881 ve->base.mask |= sibling->mask;
3882
3883
3884
3885
3886
3887
3888
3889
3890 if (ve->base.class != OTHER_CLASS) {
3891 if (ve->base.class != sibling->class) {
3892 DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n",
3893 sibling->class, ve->base.class);
3894 err = -EINVAL;
3895 goto err_put;
3896 }
3897 continue;
3898 }
3899
3900 ve->base.class = sibling->class;
3901 ve->base.uabi_class = sibling->uabi_class;
3902 snprintf(ve->base.name, sizeof(ve->base.name),
3903 "v%dx%d", ve->base.class, count);
3904 ve->base.context_size = sibling->context_size;
3905
3906 ve->base.add_active_request = sibling->add_active_request;
3907 ve->base.remove_active_request = sibling->remove_active_request;
3908 ve->base.emit_bb_start = sibling->emit_bb_start;
3909 ve->base.emit_flush = sibling->emit_flush;
3910 ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb;
3911 ve->base.emit_fini_breadcrumb = sibling->emit_fini_breadcrumb;
3912 ve->base.emit_fini_breadcrumb_dw =
3913 sibling->emit_fini_breadcrumb_dw;
3914
3915 ve->base.flags = sibling->flags;
3916 }
3917
3918 ve->base.flags |= I915_ENGINE_IS_VIRTUAL;
3919
3920 virtual_engine_initial_hint(ve);
3921 return &ve->context;
3922
3923err_put:
3924 intel_context_put(&ve->context);
3925 return ERR_PTR(err);
3926}
3927
3928void intel_execlists_show_requests(struct intel_engine_cs *engine,
3929 struct drm_printer *m,
3930 void (*show_request)(struct drm_printer *m,
3931 const struct i915_request *rq,
3932 const char *prefix,
3933 int indent),
3934 unsigned int max)
3935{
3936 const struct intel_engine_execlists *execlists = &engine->execlists;
3937 struct i915_sched_engine *sched_engine = engine->sched_engine;
3938 struct i915_request *rq, *last;
3939 unsigned long flags;
3940 unsigned int count;
3941 struct rb_node *rb;
3942
3943 spin_lock_irqsave(&sched_engine->lock, flags);
3944
3945 last = NULL;
3946 count = 0;
3947 list_for_each_entry(rq, &sched_engine->requests, sched.link) {
3948 if (count++ < max - 1)
3949 show_request(m, rq, "\t\t", 0);
3950 else
3951 last = rq;
3952 }
3953 if (last) {
3954 if (count > max) {
3955 drm_printf(m,
3956 "\t\t...skipping %d executing requests...\n",
3957 count - max);
3958 }
3959 show_request(m, last, "\t\t", 0);
3960 }
3961
3962 if (sched_engine->queue_priority_hint != INT_MIN)
3963 drm_printf(m, "\t\tQueue priority hint: %d\n",
3964 READ_ONCE(sched_engine->queue_priority_hint));
3965
3966 last = NULL;
3967 count = 0;
3968 for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) {
3969 struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
3970
3971 priolist_for_each_request(rq, p) {
3972 if (count++ < max - 1)
3973 show_request(m, rq, "\t\t", 0);
3974 else
3975 last = rq;
3976 }
3977 }
3978 if (last) {
3979 if (count > max) {
3980 drm_printf(m,
3981 "\t\t...skipping %d queued requests...\n",
3982 count - max);
3983 }
3984 show_request(m, last, "\t\t", 0);
3985 }
3986
3987 last = NULL;
3988 count = 0;
3989 for (rb = rb_first_cached(&execlists->virtual); rb; rb = rb_next(rb)) {
3990 struct virtual_engine *ve =
3991 rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
3992 struct i915_request *rq = READ_ONCE(ve->request);
3993
3994 if (rq) {
3995 if (count++ < max - 1)
3996 show_request(m, rq, "\t\t", 0);
3997 else
3998 last = rq;
3999 }
4000 }
4001 if (last) {
4002 if (count > max) {
4003 drm_printf(m,
4004 "\t\t...skipping %d virtual requests...\n",
4005 count - max);
4006 }
4007 show_request(m, last, "\t\t", 0);
4008 }
4009
4010 spin_unlock_irqrestore(&sched_engine->lock, flags);
4011}
4012
4013#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
4014#include "selftest_execlists.c"
4015#endif
4016