1
2
3
4
5
6#include <linux/prime_numbers.h>
7
8#include "gem/i915_gem_pm.h"
9#include "gt/intel_engine_heartbeat.h"
10#include "gt/intel_reset.h"
11#include "gt/selftest_engine_heartbeat.h"
12
13#include "i915_selftest.h"
14#include "selftests/i915_random.h"
15#include "selftests/igt_flush_test.h"
16#include "selftests/igt_live_test.h"
17#include "selftests/igt_spinner.h"
18#include "selftests/lib_sw_fence.h"
19
20#include "gem/selftests/igt_gem_utils.h"
21#include "gem/selftests/mock_context.h"
22
23#define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
24#define NUM_GPR 16
25#define NUM_GPR_DW (NUM_GPR * 2)
26
27static bool is_active(struct i915_request *rq)
28{
29 if (i915_request_is_active(rq))
30 return true;
31
32 if (i915_request_on_hold(rq))
33 return true;
34
35 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
36 return true;
37
38 return false;
39}
40
41static int wait_for_submit(struct intel_engine_cs *engine,
42 struct i915_request *rq,
43 unsigned long timeout)
44{
45
46 tasklet_hi_schedule(&engine->execlists.tasklet);
47
48 timeout += jiffies;
49 do {
50 bool done = time_after(jiffies, timeout);
51
52 if (i915_request_completed(rq))
53 return 0;
54
55
56 intel_engine_flush_submission(engine);
57 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
58 return 0;
59
60 if (done)
61 return -ETIME;
62
63 cond_resched();
64 } while (1);
65}
66
67static int wait_for_reset(struct intel_engine_cs *engine,
68 struct i915_request *rq,
69 unsigned long timeout)
70{
71 timeout += jiffies;
72
73 do {
74 cond_resched();
75 intel_engine_flush_submission(engine);
76
77 if (READ_ONCE(engine->execlists.pending[0]))
78 continue;
79
80 if (i915_request_completed(rq))
81 break;
82
83 if (READ_ONCE(rq->fence.error))
84 break;
85 } while (time_before(jiffies, timeout));
86
87 flush_scheduled_work();
88
89 if (rq->fence.error != -EIO) {
90 pr_err("%s: hanging request %llx:%lld not reset\n",
91 engine->name,
92 rq->fence.context,
93 rq->fence.seqno);
94 return -EINVAL;
95 }
96
97
98 if (i915_request_wait(rq, 0,
99 max(0l, (long)(timeout - jiffies)) + 1) < 0) {
100 pr_err("%s: hanging request %llx:%lld did not complete\n",
101 engine->name,
102 rq->fence.context,
103 rq->fence.seqno);
104 return -ETIME;
105 }
106
107 return 0;
108}
109
110static int live_sanitycheck(void *arg)
111{
112 struct intel_gt *gt = arg;
113 struct intel_engine_cs *engine;
114 enum intel_engine_id id;
115 struct igt_spinner spin;
116 int err = 0;
117
118 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
119 return 0;
120
121 if (igt_spinner_init(&spin, gt))
122 return -ENOMEM;
123
124 for_each_engine(engine, gt, id) {
125 struct intel_context *ce;
126 struct i915_request *rq;
127
128 ce = intel_context_create(engine);
129 if (IS_ERR(ce)) {
130 err = PTR_ERR(ce);
131 break;
132 }
133
134 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
135 if (IS_ERR(rq)) {
136 err = PTR_ERR(rq);
137 goto out_ctx;
138 }
139
140 i915_request_add(rq);
141 if (!igt_wait_for_spinner(&spin, rq)) {
142 GEM_TRACE("spinner failed to start\n");
143 GEM_TRACE_DUMP();
144 intel_gt_set_wedged(gt);
145 err = -EIO;
146 goto out_ctx;
147 }
148
149 igt_spinner_end(&spin);
150 if (igt_flush_test(gt->i915)) {
151 err = -EIO;
152 goto out_ctx;
153 }
154
155out_ctx:
156 intel_context_put(ce);
157 if (err)
158 break;
159 }
160
161 igt_spinner_fini(&spin);
162 return err;
163}
164
165static int live_unlite_restore(struct intel_gt *gt, int prio)
166{
167 struct intel_engine_cs *engine;
168 enum intel_engine_id id;
169 struct igt_spinner spin;
170 int err = -ENOMEM;
171
172
173
174
175
176
177 if (igt_spinner_init(&spin, gt))
178 return err;
179
180 err = 0;
181 for_each_engine(engine, gt, id) {
182 struct intel_context *ce[2] = {};
183 struct i915_request *rq[2];
184 struct igt_live_test t;
185 int n;
186
187 if (prio && !intel_engine_has_preemption(engine))
188 continue;
189
190 if (!intel_engine_can_store_dword(engine))
191 continue;
192
193 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
194 err = -EIO;
195 break;
196 }
197 st_engine_heartbeat_disable(engine);
198
199 for (n = 0; n < ARRAY_SIZE(ce); n++) {
200 struct intel_context *tmp;
201
202 tmp = intel_context_create(engine);
203 if (IS_ERR(tmp)) {
204 err = PTR_ERR(tmp);
205 goto err_ce;
206 }
207
208 err = intel_context_pin(tmp);
209 if (err) {
210 intel_context_put(tmp);
211 goto err_ce;
212 }
213
214
215
216
217
218
219 memset(tmp->ring->vaddr,
220 POISON_INUSE,
221 tmp->ring->vma->size);
222
223 ce[n] = tmp;
224 }
225 GEM_BUG_ON(!ce[1]->ring->size);
226 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
227 lrc_update_regs(ce[1], engine, ce[1]->ring->head);
228
229 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
230 if (IS_ERR(rq[0])) {
231 err = PTR_ERR(rq[0]);
232 goto err_ce;
233 }
234
235 i915_request_get(rq[0]);
236 i915_request_add(rq[0]);
237 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
238
239 if (!igt_wait_for_spinner(&spin, rq[0])) {
240 i915_request_put(rq[0]);
241 goto err_ce;
242 }
243
244 rq[1] = i915_request_create(ce[1]);
245 if (IS_ERR(rq[1])) {
246 err = PTR_ERR(rq[1]);
247 i915_request_put(rq[0]);
248 goto err_ce;
249 }
250
251 if (!prio) {
252
253
254
255
256
257
258
259
260
261
262 i915_request_await_dma_fence(rq[1], &rq[0]->fence);
263 }
264
265 i915_request_get(rq[1]);
266 i915_request_add(rq[1]);
267 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
268 i915_request_put(rq[0]);
269
270 if (prio) {
271 struct i915_sched_attr attr = {
272 .priority = prio,
273 };
274
275
276 engine->schedule(rq[1], &attr);
277 }
278
279
280 rq[0] = i915_request_create(ce[0]);
281 if (IS_ERR(rq[0])) {
282 err = PTR_ERR(rq[0]);
283 i915_request_put(rq[1]);
284 goto err_ce;
285 }
286
287 i915_request_await_dma_fence(rq[0], &rq[1]->fence);
288 i915_request_get(rq[0]);
289 i915_request_add(rq[0]);
290 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
291 i915_request_put(rq[1]);
292 i915_request_put(rq[0]);
293
294err_ce:
295 intel_engine_flush_submission(engine);
296 igt_spinner_end(&spin);
297 for (n = 0; n < ARRAY_SIZE(ce); n++) {
298 if (IS_ERR_OR_NULL(ce[n]))
299 break;
300
301 intel_context_unpin(ce[n]);
302 intel_context_put(ce[n]);
303 }
304
305 st_engine_heartbeat_enable(engine);
306 if (igt_live_test_end(&t))
307 err = -EIO;
308 if (err)
309 break;
310 }
311
312 igt_spinner_fini(&spin);
313 return err;
314}
315
316static int live_unlite_switch(void *arg)
317{
318 return live_unlite_restore(arg, 0);
319}
320
321static int live_unlite_preempt(void *arg)
322{
323 return live_unlite_restore(arg, I915_PRIORITY_MAX);
324}
325
326static int live_unlite_ring(void *arg)
327{
328 struct intel_gt *gt = arg;
329 struct intel_engine_cs *engine;
330 struct igt_spinner spin;
331 enum intel_engine_id id;
332 int err = 0;
333
334
335
336
337
338
339
340 if (igt_spinner_init(&spin, gt))
341 return -ENOMEM;
342
343 for_each_engine(engine, gt, id) {
344 struct intel_context *ce[2] = {};
345 struct i915_request *rq;
346 struct igt_live_test t;
347 int n;
348
349 if (!intel_engine_has_preemption(engine))
350 continue;
351
352 if (!intel_engine_can_store_dword(engine))
353 continue;
354
355 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
356 err = -EIO;
357 break;
358 }
359 st_engine_heartbeat_disable(engine);
360
361 for (n = 0; n < ARRAY_SIZE(ce); n++) {
362 struct intel_context *tmp;
363
364 tmp = intel_context_create(engine);
365 if (IS_ERR(tmp)) {
366 err = PTR_ERR(tmp);
367 goto err_ce;
368 }
369
370 err = intel_context_pin(tmp);
371 if (err) {
372 intel_context_put(tmp);
373 goto err_ce;
374 }
375
376 memset32(tmp->ring->vaddr,
377 0xdeadbeef,
378 tmp->ring->vma->size / sizeof(u32));
379
380 ce[n] = tmp;
381 }
382
383
384 rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
385 if (IS_ERR(rq)) {
386 err = PTR_ERR(rq);
387 goto err_ce;
388 }
389
390 i915_request_get(rq);
391 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
392 i915_request_add(rq);
393
394 if (!igt_wait_for_spinner(&spin, rq)) {
395 intel_gt_set_wedged(gt);
396 i915_request_put(rq);
397 err = -ETIME;
398 goto err_ce;
399 }
400
401
402 n = 0;
403 while (intel_ring_direction(ce[0]->ring,
404 rq->wa_tail,
405 ce[0]->ring->tail) <= 0) {
406 struct i915_request *tmp;
407
408 tmp = intel_context_create_request(ce[0]);
409 if (IS_ERR(tmp)) {
410 err = PTR_ERR(tmp);
411 i915_request_put(rq);
412 goto err_ce;
413 }
414
415 i915_request_add(tmp);
416 intel_engine_flush_submission(engine);
417 n++;
418 }
419 intel_engine_flush_submission(engine);
420 pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
421 engine->name, n,
422 ce[0]->ring->size,
423 ce[0]->ring->tail,
424 ce[0]->ring->emit,
425 rq->tail);
426 GEM_BUG_ON(intel_ring_direction(ce[0]->ring,
427 rq->tail,
428 ce[0]->ring->tail) <= 0);
429 i915_request_put(rq);
430
431
432 rq = intel_context_create_request(ce[1]);
433 if (IS_ERR(rq)) {
434 err = PTR_ERR(rq);
435 goto err_ce;
436 }
437
438 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
439 i915_request_get(rq);
440 i915_request_add(rq);
441
442 err = wait_for_submit(engine, rq, HZ / 2);
443 i915_request_put(rq);
444 if (err) {
445 pr_err("%s: preemption request was not submitted\n",
446 engine->name);
447 err = -ETIME;
448 }
449
450 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
451 engine->name,
452 ce[0]->ring->tail, ce[0]->ring->emit,
453 ce[1]->ring->tail, ce[1]->ring->emit);
454
455err_ce:
456 intel_engine_flush_submission(engine);
457 igt_spinner_end(&spin);
458 for (n = 0; n < ARRAY_SIZE(ce); n++) {
459 if (IS_ERR_OR_NULL(ce[n]))
460 break;
461
462 intel_context_unpin(ce[n]);
463 intel_context_put(ce[n]);
464 }
465 st_engine_heartbeat_enable(engine);
466 if (igt_live_test_end(&t))
467 err = -EIO;
468 if (err)
469 break;
470 }
471
472 igt_spinner_fini(&spin);
473 return err;
474}
475
476static int live_pin_rewind(void *arg)
477{
478 struct intel_gt *gt = arg;
479 struct intel_engine_cs *engine;
480 enum intel_engine_id id;
481 int err = 0;
482
483
484
485
486
487
488
489
490
491
492 for_each_engine(engine, gt, id) {
493 struct intel_context *ce;
494 struct i915_request *rq;
495 struct intel_ring *ring;
496 struct igt_live_test t;
497
498 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
499 err = -EIO;
500 break;
501 }
502
503 ce = intel_context_create(engine);
504 if (IS_ERR(ce)) {
505 err = PTR_ERR(ce);
506 break;
507 }
508
509 err = intel_context_pin(ce);
510 if (err) {
511 intel_context_put(ce);
512 break;
513 }
514
515
516 err = i915_active_acquire(&ce->active);
517 if (err) {
518 intel_context_unpin(ce);
519 intel_context_put(ce);
520 break;
521 }
522 ring = ce->ring;
523
524
525 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
526 ring->emit = ring->size / 2;
527 ring->tail = ring->emit;
528 GEM_BUG_ON(ring->head);
529
530 intel_context_unpin(ce);
531
532
533 GEM_BUG_ON(intel_context_is_pinned(ce));
534 rq = intel_context_create_request(ce);
535 i915_active_release(&ce->active);
536 intel_context_put(ce);
537 if (IS_ERR(rq)) {
538 err = PTR_ERR(rq);
539 break;
540 }
541 GEM_BUG_ON(!rq->head);
542 i915_request_add(rq);
543
544
545 if (igt_live_test_end(&t)) {
546 err = -EIO;
547 break;
548 }
549 }
550
551 return err;
552}
553
554static int engine_lock_reset_tasklet(struct intel_engine_cs *engine)
555{
556 tasklet_disable(&engine->execlists.tasklet);
557 local_bh_disable();
558
559 if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
560 &engine->gt->reset.flags)) {
561 local_bh_enable();
562 tasklet_enable(&engine->execlists.tasklet);
563
564 intel_gt_set_wedged(engine->gt);
565 return -EBUSY;
566 }
567
568 return 0;
569}
570
571static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine)
572{
573 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,
574 &engine->gt->reset.flags);
575
576 local_bh_enable();
577 tasklet_enable(&engine->execlists.tasklet);
578}
579
580static int live_hold_reset(void *arg)
581{
582 struct intel_gt *gt = arg;
583 struct intel_engine_cs *engine;
584 enum intel_engine_id id;
585 struct igt_spinner spin;
586 int err = 0;
587
588
589
590
591
592
593
594 if (!intel_has_reset_engine(gt))
595 return 0;
596
597 if (igt_spinner_init(&spin, gt))
598 return -ENOMEM;
599
600 for_each_engine(engine, gt, id) {
601 struct intel_context *ce;
602 struct i915_request *rq;
603
604 ce = intel_context_create(engine);
605 if (IS_ERR(ce)) {
606 err = PTR_ERR(ce);
607 break;
608 }
609
610 st_engine_heartbeat_disable(engine);
611
612 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
613 if (IS_ERR(rq)) {
614 err = PTR_ERR(rq);
615 goto out;
616 }
617 i915_request_add(rq);
618
619 if (!igt_wait_for_spinner(&spin, rq)) {
620 intel_gt_set_wedged(gt);
621 err = -ETIME;
622 goto out;
623 }
624
625
626
627 err = engine_lock_reset_tasklet(engine);
628 if (err)
629 goto out;
630
631 engine->execlists.tasklet.callback(&engine->execlists.tasklet);
632 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
633
634 i915_request_get(rq);
635 execlists_hold(engine, rq);
636 GEM_BUG_ON(!i915_request_on_hold(rq));
637
638 __intel_engine_reset_bh(engine, NULL);
639 GEM_BUG_ON(rq->fence.error != -EIO);
640
641 engine_unlock_reset_tasklet(engine);
642
643
644 if (!i915_request_wait(rq, 0, HZ / 5)) {
645 pr_err("%s: on hold request completed!\n",
646 engine->name);
647 i915_request_put(rq);
648 err = -EIO;
649 goto out;
650 }
651 GEM_BUG_ON(!i915_request_on_hold(rq));
652
653
654 execlists_unhold(engine, rq);
655 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
656 pr_err("%s: held request did not complete!\n",
657 engine->name);
658 intel_gt_set_wedged(gt);
659 err = -ETIME;
660 }
661 i915_request_put(rq);
662
663out:
664 st_engine_heartbeat_enable(engine);
665 intel_context_put(ce);
666 if (err)
667 break;
668 }
669
670 igt_spinner_fini(&spin);
671 return err;
672}
673
674static const char *error_repr(int err)
675{
676 return err ? "bad" : "good";
677}
678
679static int live_error_interrupt(void *arg)
680{
681 static const struct error_phase {
682 enum { GOOD = 0, BAD = -EIO } error[2];
683 } phases[] = {
684 { { BAD, GOOD } },
685 { { BAD, BAD } },
686 { { BAD, GOOD } },
687 { { GOOD, GOOD } },
688 };
689 struct intel_gt *gt = arg;
690 struct intel_engine_cs *engine;
691 enum intel_engine_id id;
692
693
694
695
696
697
698
699
700
701
702
703
704 if (!intel_has_reset_engine(gt))
705 return 0;
706
707 for_each_engine(engine, gt, id) {
708 const struct error_phase *p;
709 int err = 0;
710
711 st_engine_heartbeat_disable(engine);
712
713 for (p = phases; p->error[0] != GOOD; p++) {
714 struct i915_request *client[ARRAY_SIZE(phases->error)];
715 u32 *cs;
716 int i;
717
718 memset(client, 0, sizeof(*client));
719 for (i = 0; i < ARRAY_SIZE(client); i++) {
720 struct intel_context *ce;
721 struct i915_request *rq;
722
723 ce = intel_context_create(engine);
724 if (IS_ERR(ce)) {
725 err = PTR_ERR(ce);
726 goto out;
727 }
728
729 rq = intel_context_create_request(ce);
730 intel_context_put(ce);
731 if (IS_ERR(rq)) {
732 err = PTR_ERR(rq);
733 goto out;
734 }
735
736 if (rq->engine->emit_init_breadcrumb) {
737 err = rq->engine->emit_init_breadcrumb(rq);
738 if (err) {
739 i915_request_add(rq);
740 goto out;
741 }
742 }
743
744 cs = intel_ring_begin(rq, 2);
745 if (IS_ERR(cs)) {
746 i915_request_add(rq);
747 err = PTR_ERR(cs);
748 goto out;
749 }
750
751 if (p->error[i]) {
752 *cs++ = 0xdeadbeef;
753 *cs++ = 0xdeadbeef;
754 } else {
755 *cs++ = MI_NOOP;
756 *cs++ = MI_NOOP;
757 }
758
759 client[i] = i915_request_get(rq);
760 i915_request_add(rq);
761 }
762
763 err = wait_for_submit(engine, client[0], HZ / 2);
764 if (err) {
765 pr_err("%s: first request did not start within time!\n",
766 engine->name);
767 err = -ETIME;
768 goto out;
769 }
770
771 for (i = 0; i < ARRAY_SIZE(client); i++) {
772 if (i915_request_wait(client[i], 0, HZ / 5) < 0)
773 pr_debug("%s: %s request incomplete!\n",
774 engine->name,
775 error_repr(p->error[i]));
776
777 if (!i915_request_started(client[i])) {
778 pr_err("%s: %s request not started!\n",
779 engine->name,
780 error_repr(p->error[i]));
781 err = -ETIME;
782 goto out;
783 }
784
785
786 intel_engine_flush_submission(engine);
787 if (client[i]->fence.error != p->error[i]) {
788 pr_err("%s: %s request (%s) with wrong error code: %d\n",
789 engine->name,
790 error_repr(p->error[i]),
791 i915_request_completed(client[i]) ? "completed" : "running",
792 client[i]->fence.error);
793 err = -EINVAL;
794 goto out;
795 }
796 }
797
798out:
799 for (i = 0; i < ARRAY_SIZE(client); i++)
800 if (client[i])
801 i915_request_put(client[i]);
802 if (err) {
803 pr_err("%s: failed at phase[%zd] { %d, %d }\n",
804 engine->name, p - phases,
805 p->error[0], p->error[1]);
806 break;
807 }
808 }
809
810 st_engine_heartbeat_enable(engine);
811 if (err) {
812 intel_gt_set_wedged(gt);
813 return err;
814 }
815 }
816
817 return 0;
818}
819
820static int
821emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
822{
823 u32 *cs;
824
825 cs = intel_ring_begin(rq, 10);
826 if (IS_ERR(cs))
827 return PTR_ERR(cs);
828
829 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
830
831 *cs++ = MI_SEMAPHORE_WAIT |
832 MI_SEMAPHORE_GLOBAL_GTT |
833 MI_SEMAPHORE_POLL |
834 MI_SEMAPHORE_SAD_NEQ_SDD;
835 *cs++ = 0;
836 *cs++ = i915_ggtt_offset(vma) + 4 * idx;
837 *cs++ = 0;
838
839 if (idx > 0) {
840 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
841 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
842 *cs++ = 0;
843 *cs++ = 1;
844 } else {
845 *cs++ = MI_NOOP;
846 *cs++ = MI_NOOP;
847 *cs++ = MI_NOOP;
848 *cs++ = MI_NOOP;
849 }
850
851 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
852
853 intel_ring_advance(rq, cs);
854 return 0;
855}
856
857static struct i915_request *
858semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
859{
860 struct intel_context *ce;
861 struct i915_request *rq;
862 int err;
863
864 ce = intel_context_create(engine);
865 if (IS_ERR(ce))
866 return ERR_CAST(ce);
867
868 rq = intel_context_create_request(ce);
869 if (IS_ERR(rq))
870 goto out_ce;
871
872 err = 0;
873 if (rq->engine->emit_init_breadcrumb)
874 err = rq->engine->emit_init_breadcrumb(rq);
875 if (err == 0)
876 err = emit_semaphore_chain(rq, vma, idx);
877 if (err == 0)
878 i915_request_get(rq);
879 i915_request_add(rq);
880 if (err)
881 rq = ERR_PTR(err);
882
883out_ce:
884 intel_context_put(ce);
885 return rq;
886}
887
888static int
889release_queue(struct intel_engine_cs *engine,
890 struct i915_vma *vma,
891 int idx, int prio)
892{
893 struct i915_sched_attr attr = {
894 .priority = prio,
895 };
896 struct i915_request *rq;
897 u32 *cs;
898
899 rq = intel_engine_create_kernel_request(engine);
900 if (IS_ERR(rq))
901 return PTR_ERR(rq);
902
903 cs = intel_ring_begin(rq, 4);
904 if (IS_ERR(cs)) {
905 i915_request_add(rq);
906 return PTR_ERR(cs);
907 }
908
909 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
910 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
911 *cs++ = 0;
912 *cs++ = 1;
913
914 intel_ring_advance(rq, cs);
915
916 i915_request_get(rq);
917 i915_request_add(rq);
918
919 local_bh_disable();
920 engine->schedule(rq, &attr);
921 local_bh_enable();
922
923 i915_request_put(rq);
924
925 return 0;
926}
927
928static int
929slice_semaphore_queue(struct intel_engine_cs *outer,
930 struct i915_vma *vma,
931 int count)
932{
933 struct intel_engine_cs *engine;
934 struct i915_request *head;
935 enum intel_engine_id id;
936 int err, i, n = 0;
937
938 head = semaphore_queue(outer, vma, n++);
939 if (IS_ERR(head))
940 return PTR_ERR(head);
941
942 for_each_engine(engine, outer->gt, id) {
943 if (!intel_engine_has_preemption(engine))
944 continue;
945
946 for (i = 0; i < count; i++) {
947 struct i915_request *rq;
948
949 rq = semaphore_queue(engine, vma, n++);
950 if (IS_ERR(rq)) {
951 err = PTR_ERR(rq);
952 goto out;
953 }
954
955 i915_request_put(rq);
956 }
957 }
958
959 err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
960 if (err)
961 goto out;
962
963 if (i915_request_wait(head, 0,
964 2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) {
965 pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n",
966 outer->name, count, n);
967 GEM_TRACE_DUMP();
968 intel_gt_set_wedged(outer->gt);
969 err = -EIO;
970 }
971
972out:
973 i915_request_put(head);
974 return err;
975}
976
977static int live_timeslice_preempt(void *arg)
978{
979 struct intel_gt *gt = arg;
980 struct drm_i915_gem_object *obj;
981 struct intel_engine_cs *engine;
982 enum intel_engine_id id;
983 struct i915_vma *vma;
984 void *vaddr;
985 int err = 0;
986
987
988
989
990
991
992
993
994
995 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
996 return 0;
997
998 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
999 if (IS_ERR(obj))
1000 return PTR_ERR(obj);
1001
1002 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1003 if (IS_ERR(vma)) {
1004 err = PTR_ERR(vma);
1005 goto err_obj;
1006 }
1007
1008 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1009 if (IS_ERR(vaddr)) {
1010 err = PTR_ERR(vaddr);
1011 goto err_obj;
1012 }
1013
1014 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1015 if (err)
1016 goto err_map;
1017
1018 err = i915_vma_sync(vma);
1019 if (err)
1020 goto err_pin;
1021
1022 for_each_engine(engine, gt, id) {
1023 if (!intel_engine_has_preemption(engine))
1024 continue;
1025
1026 memset(vaddr, 0, PAGE_SIZE);
1027
1028 st_engine_heartbeat_disable(engine);
1029 err = slice_semaphore_queue(engine, vma, 5);
1030 st_engine_heartbeat_enable(engine);
1031 if (err)
1032 goto err_pin;
1033
1034 if (igt_flush_test(gt->i915)) {
1035 err = -EIO;
1036 goto err_pin;
1037 }
1038 }
1039
1040err_pin:
1041 i915_vma_unpin(vma);
1042err_map:
1043 i915_gem_object_unpin_map(obj);
1044err_obj:
1045 i915_gem_object_put(obj);
1046 return err;
1047}
1048
1049static struct i915_request *
1050create_rewinder(struct intel_context *ce,
1051 struct i915_request *wait,
1052 void *slot, int idx)
1053{
1054 const u32 offset =
1055 i915_ggtt_offset(ce->engine->status_page.vma) +
1056 offset_in_page(slot);
1057 struct i915_request *rq;
1058 u32 *cs;
1059 int err;
1060
1061 rq = intel_context_create_request(ce);
1062 if (IS_ERR(rq))
1063 return rq;
1064
1065 if (wait) {
1066 err = i915_request_await_dma_fence(rq, &wait->fence);
1067 if (err)
1068 goto err;
1069 }
1070
1071 cs = intel_ring_begin(rq, 14);
1072 if (IS_ERR(cs)) {
1073 err = PTR_ERR(cs);
1074 goto err;
1075 }
1076
1077 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1078 *cs++ = MI_NOOP;
1079
1080 *cs++ = MI_SEMAPHORE_WAIT |
1081 MI_SEMAPHORE_GLOBAL_GTT |
1082 MI_SEMAPHORE_POLL |
1083 MI_SEMAPHORE_SAD_GTE_SDD;
1084 *cs++ = idx;
1085 *cs++ = offset;
1086 *cs++ = 0;
1087
1088 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1089 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
1090 *cs++ = offset + idx * sizeof(u32);
1091 *cs++ = 0;
1092
1093 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1094 *cs++ = offset;
1095 *cs++ = 0;
1096 *cs++ = idx + 1;
1097
1098 intel_ring_advance(rq, cs);
1099
1100 err = 0;
1101err:
1102 i915_request_get(rq);
1103 i915_request_add(rq);
1104 if (err) {
1105 i915_request_put(rq);
1106 return ERR_PTR(err);
1107 }
1108
1109 return rq;
1110}
1111
1112static int live_timeslice_rewind(void *arg)
1113{
1114 struct intel_gt *gt = arg;
1115 struct intel_engine_cs *engine;
1116 enum intel_engine_id id;
1117
1118
1119
1120
1121
1122
1123
1124
1125 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1126 return 0;
1127
1128 for_each_engine(engine, gt, id) {
1129 enum { A1, A2, B1 };
1130 enum { X = 1, Z, Y };
1131 struct i915_request *rq[3] = {};
1132 struct intel_context *ce;
1133 unsigned long timeslice;
1134 int i, err = 0;
1135 u32 *slot;
1136
1137 if (!intel_engine_has_timeslices(engine))
1138 continue;
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151 st_engine_heartbeat_disable(engine);
1152 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1153
1154 slot = memset32(engine->status_page.addr + 1000, 0, 4);
1155
1156 ce = intel_context_create(engine);
1157 if (IS_ERR(ce)) {
1158 err = PTR_ERR(ce);
1159 goto err;
1160 }
1161
1162 rq[A1] = create_rewinder(ce, NULL, slot, X);
1163 if (IS_ERR(rq[A1])) {
1164 intel_context_put(ce);
1165 goto err;
1166 }
1167
1168 rq[A2] = create_rewinder(ce, NULL, slot, Y);
1169 intel_context_put(ce);
1170 if (IS_ERR(rq[A2]))
1171 goto err;
1172
1173 err = wait_for_submit(engine, rq[A2], HZ / 2);
1174 if (err) {
1175 pr_err("%s: failed to submit first context\n",
1176 engine->name);
1177 goto err;
1178 }
1179
1180 ce = intel_context_create(engine);
1181 if (IS_ERR(ce)) {
1182 err = PTR_ERR(ce);
1183 goto err;
1184 }
1185
1186 rq[B1] = create_rewinder(ce, rq[A1], slot, Z);
1187 intel_context_put(ce);
1188 if (IS_ERR(rq[2]))
1189 goto err;
1190
1191 err = wait_for_submit(engine, rq[B1], HZ / 2);
1192 if (err) {
1193 pr_err("%s: failed to submit second context\n",
1194 engine->name);
1195 goto err;
1196 }
1197
1198
1199 ENGINE_TRACE(engine, "forcing tasklet for rewind\n");
1200 while (i915_request_is_active(rq[A2])) {
1201
1202 del_timer(&engine->execlists.timer);
1203 tasklet_hi_schedule(&engine->execlists.tasklet);
1204 intel_engine_flush_submission(engine);
1205 }
1206
1207 GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1208 GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1209 GEM_BUG_ON(i915_request_is_active(rq[A2]));
1210
1211
1212 slot[0] = 1;
1213 wmb();
1214
1215 for (i = 1; i <= 3; i++) {
1216 unsigned long timeout = jiffies + HZ / 2;
1217
1218 while (!READ_ONCE(slot[i]) &&
1219 time_before(jiffies, timeout))
1220 ;
1221
1222 if (!time_before(jiffies, timeout)) {
1223 pr_err("%s: rq[%d] timed out\n",
1224 engine->name, i - 1);
1225 err = -ETIME;
1226 goto err;
1227 }
1228
1229 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1230 }
1231
1232
1233 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1234 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1235 engine->name,
1236 slot[Z] - slot[X],
1237 slot[Y] - slot[X]);
1238 err = -EINVAL;
1239 }
1240
1241err:
1242 memset32(&slot[0], -1, 4);
1243 wmb();
1244
1245 engine->props.timeslice_duration_ms = timeslice;
1246 st_engine_heartbeat_enable(engine);
1247 for (i = 0; i < 3; i++)
1248 i915_request_put(rq[i]);
1249 if (igt_flush_test(gt->i915))
1250 err = -EIO;
1251 if (err)
1252 return err;
1253 }
1254
1255 return 0;
1256}
1257
1258static struct i915_request *nop_request(struct intel_engine_cs *engine)
1259{
1260 struct i915_request *rq;
1261
1262 rq = intel_engine_create_kernel_request(engine);
1263 if (IS_ERR(rq))
1264 return rq;
1265
1266 i915_request_get(rq);
1267 i915_request_add(rq);
1268
1269 return rq;
1270}
1271
1272static long slice_timeout(struct intel_engine_cs *engine)
1273{
1274 long timeout;
1275
1276
1277 timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine));
1278
1279
1280 timeout += HZ / 5;
1281
1282 return timeout + 1;
1283}
1284
1285static int live_timeslice_queue(void *arg)
1286{
1287 struct intel_gt *gt = arg;
1288 struct drm_i915_gem_object *obj;
1289 struct intel_engine_cs *engine;
1290 enum intel_engine_id id;
1291 struct i915_vma *vma;
1292 void *vaddr;
1293 int err = 0;
1294
1295
1296
1297
1298
1299
1300
1301
1302 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1303 return 0;
1304
1305 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1306 if (IS_ERR(obj))
1307 return PTR_ERR(obj);
1308
1309 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1310 if (IS_ERR(vma)) {
1311 err = PTR_ERR(vma);
1312 goto err_obj;
1313 }
1314
1315 vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1316 if (IS_ERR(vaddr)) {
1317 err = PTR_ERR(vaddr);
1318 goto err_obj;
1319 }
1320
1321 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1322 if (err)
1323 goto err_map;
1324
1325 err = i915_vma_sync(vma);
1326 if (err)
1327 goto err_pin;
1328
1329 for_each_engine(engine, gt, id) {
1330 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
1331 struct i915_request *rq, *nop;
1332
1333 if (!intel_engine_has_preemption(engine))
1334 continue;
1335
1336 st_engine_heartbeat_disable(engine);
1337 memset(vaddr, 0, PAGE_SIZE);
1338
1339
1340 rq = semaphore_queue(engine, vma, 0);
1341 if (IS_ERR(rq)) {
1342 err = PTR_ERR(rq);
1343 goto err_heartbeat;
1344 }
1345 engine->schedule(rq, &attr);
1346 err = wait_for_submit(engine, rq, HZ / 2);
1347 if (err) {
1348 pr_err("%s: Timed out trying to submit semaphores\n",
1349 engine->name);
1350 goto err_rq;
1351 }
1352
1353
1354 nop = nop_request(engine);
1355 if (IS_ERR(nop)) {
1356 err = PTR_ERR(nop);
1357 goto err_rq;
1358 }
1359 err = wait_for_submit(engine, nop, HZ / 2);
1360 i915_request_put(nop);
1361 if (err) {
1362 pr_err("%s: Timed out trying to submit nop\n",
1363 engine->name);
1364 goto err_rq;
1365 }
1366
1367 GEM_BUG_ON(i915_request_completed(rq));
1368 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1369
1370
1371 err = release_queue(engine, vma, 1, effective_prio(rq));
1372 if (err)
1373 goto err_rq;
1374
1375
1376 do {
1377 cond_resched();
1378 intel_engine_flush_submission(engine);
1379 } while (READ_ONCE(engine->execlists.pending[0]));
1380
1381
1382 if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) {
1383 struct drm_printer p =
1384 drm_info_printer(gt->i915->drm.dev);
1385
1386 pr_err("%s: Failed to timeslice into queue\n",
1387 engine->name);
1388 intel_engine_dump(engine, &p,
1389 "%s\n", engine->name);
1390
1391 memset(vaddr, 0xff, PAGE_SIZE);
1392 err = -EIO;
1393 }
1394err_rq:
1395 i915_request_put(rq);
1396err_heartbeat:
1397 st_engine_heartbeat_enable(engine);
1398 if (err)
1399 break;
1400 }
1401
1402err_pin:
1403 i915_vma_unpin(vma);
1404err_map:
1405 i915_gem_object_unpin_map(obj);
1406err_obj:
1407 i915_gem_object_put(obj);
1408 return err;
1409}
1410
1411static int live_timeslice_nopreempt(void *arg)
1412{
1413 struct intel_gt *gt = arg;
1414 struct intel_engine_cs *engine;
1415 enum intel_engine_id id;
1416 struct igt_spinner spin;
1417 int err = 0;
1418
1419
1420
1421
1422
1423 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1424 return 0;
1425
1426 if (igt_spinner_init(&spin, gt))
1427 return -ENOMEM;
1428
1429 for_each_engine(engine, gt, id) {
1430 struct intel_context *ce;
1431 struct i915_request *rq;
1432 unsigned long timeslice;
1433
1434 if (!intel_engine_has_preemption(engine))
1435 continue;
1436
1437 ce = intel_context_create(engine);
1438 if (IS_ERR(ce)) {
1439 err = PTR_ERR(ce);
1440 break;
1441 }
1442
1443 st_engine_heartbeat_disable(engine);
1444 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1445
1446
1447
1448 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
1449 intel_context_put(ce);
1450 if (IS_ERR(rq)) {
1451 err = PTR_ERR(rq);
1452 goto out_heartbeat;
1453 }
1454
1455 i915_request_get(rq);
1456 i915_request_add(rq);
1457
1458 if (!igt_wait_for_spinner(&spin, rq)) {
1459 i915_request_put(rq);
1460 err = -ETIME;
1461 goto out_spin;
1462 }
1463
1464 set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
1465 i915_request_put(rq);
1466
1467
1468
1469 ce = intel_context_create(engine);
1470 if (IS_ERR(ce)) {
1471 err = PTR_ERR(ce);
1472 goto out_spin;
1473 }
1474
1475 rq = intel_context_create_request(ce);
1476 intel_context_put(ce);
1477 if (IS_ERR(rq)) {
1478 err = PTR_ERR(rq);
1479 goto out_spin;
1480 }
1481
1482 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1483 i915_request_get(rq);
1484 i915_request_add(rq);
1485
1486
1487
1488
1489
1490 if (wait_for_submit(engine, rq, HZ / 2)) {
1491 i915_request_put(rq);
1492 err = -ETIME;
1493 goto out_spin;
1494 }
1495
1496
1497
1498
1499
1500
1501 if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) {
1502 pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",
1503 engine->name);
1504 err = -EINVAL;
1505 }
1506 i915_request_put(rq);
1507
1508out_spin:
1509 igt_spinner_end(&spin);
1510out_heartbeat:
1511 xchg(&engine->props.timeslice_duration_ms, timeslice);
1512 st_engine_heartbeat_enable(engine);
1513 if (err)
1514 break;
1515
1516 if (igt_flush_test(gt->i915)) {
1517 err = -EIO;
1518 break;
1519 }
1520 }
1521
1522 igt_spinner_fini(&spin);
1523 return err;
1524}
1525
1526static int live_busywait_preempt(void *arg)
1527{
1528 struct intel_gt *gt = arg;
1529 struct i915_gem_context *ctx_hi, *ctx_lo;
1530 struct intel_engine_cs *engine;
1531 struct drm_i915_gem_object *obj;
1532 struct i915_vma *vma;
1533 enum intel_engine_id id;
1534 int err = -ENOMEM;
1535 u32 *map;
1536
1537
1538
1539
1540
1541
1542 ctx_hi = kernel_context(gt->i915);
1543 if (!ctx_hi)
1544 return -ENOMEM;
1545 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1546
1547 ctx_lo = kernel_context(gt->i915);
1548 if (!ctx_lo)
1549 goto err_ctx_hi;
1550 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1551
1552 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1553 if (IS_ERR(obj)) {
1554 err = PTR_ERR(obj);
1555 goto err_ctx_lo;
1556 }
1557
1558 map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1559 if (IS_ERR(map)) {
1560 err = PTR_ERR(map);
1561 goto err_obj;
1562 }
1563
1564 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
1565 if (IS_ERR(vma)) {
1566 err = PTR_ERR(vma);
1567 goto err_map;
1568 }
1569
1570 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1571 if (err)
1572 goto err_map;
1573
1574 err = i915_vma_sync(vma);
1575 if (err)
1576 goto err_vma;
1577
1578 for_each_engine(engine, gt, id) {
1579 struct i915_request *lo, *hi;
1580 struct igt_live_test t;
1581 u32 *cs;
1582
1583 if (!intel_engine_has_preemption(engine))
1584 continue;
1585
1586 if (!intel_engine_can_store_dword(engine))
1587 continue;
1588
1589 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1590 err = -EIO;
1591 goto err_vma;
1592 }
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603 lo = igt_request_alloc(ctx_lo, engine);
1604 if (IS_ERR(lo)) {
1605 err = PTR_ERR(lo);
1606 goto err_vma;
1607 }
1608
1609 cs = intel_ring_begin(lo, 8);
1610 if (IS_ERR(cs)) {
1611 err = PTR_ERR(cs);
1612 i915_request_add(lo);
1613 goto err_vma;
1614 }
1615
1616 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1617 *cs++ = i915_ggtt_offset(vma);
1618 *cs++ = 0;
1619 *cs++ = 1;
1620
1621
1622
1623 *cs++ = MI_SEMAPHORE_WAIT |
1624 MI_SEMAPHORE_GLOBAL_GTT |
1625 MI_SEMAPHORE_POLL |
1626 MI_SEMAPHORE_SAD_EQ_SDD;
1627 *cs++ = 0;
1628 *cs++ = i915_ggtt_offset(vma);
1629 *cs++ = 0;
1630
1631 intel_ring_advance(lo, cs);
1632
1633 i915_request_get(lo);
1634 i915_request_add(lo);
1635
1636 if (wait_for(READ_ONCE(*map), 10)) {
1637 i915_request_put(lo);
1638 err = -ETIMEDOUT;
1639 goto err_vma;
1640 }
1641
1642
1643 if (i915_request_wait(lo, 0, 1) != -ETIME) {
1644 i915_request_put(lo);
1645 pr_err("%s: Busywaiting request did not!\n",
1646 engine->name);
1647 err = -EIO;
1648 goto err_vma;
1649 }
1650
1651 hi = igt_request_alloc(ctx_hi, engine);
1652 if (IS_ERR(hi)) {
1653 err = PTR_ERR(hi);
1654 i915_request_put(lo);
1655 goto err_vma;
1656 }
1657
1658 cs = intel_ring_begin(hi, 4);
1659 if (IS_ERR(cs)) {
1660 err = PTR_ERR(cs);
1661 i915_request_add(hi);
1662 i915_request_put(lo);
1663 goto err_vma;
1664 }
1665
1666 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1667 *cs++ = i915_ggtt_offset(vma);
1668 *cs++ = 0;
1669 *cs++ = 0;
1670
1671 intel_ring_advance(hi, cs);
1672 i915_request_add(hi);
1673
1674 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1675 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
1676
1677 pr_err("%s: Failed to preempt semaphore busywait!\n",
1678 engine->name);
1679
1680 intel_engine_dump(engine, &p, "%s\n", engine->name);
1681 GEM_TRACE_DUMP();
1682
1683 i915_request_put(lo);
1684 intel_gt_set_wedged(gt);
1685 err = -EIO;
1686 goto err_vma;
1687 }
1688 GEM_BUG_ON(READ_ONCE(*map));
1689 i915_request_put(lo);
1690
1691 if (igt_live_test_end(&t)) {
1692 err = -EIO;
1693 goto err_vma;
1694 }
1695 }
1696
1697 err = 0;
1698err_vma:
1699 i915_vma_unpin(vma);
1700err_map:
1701 i915_gem_object_unpin_map(obj);
1702err_obj:
1703 i915_gem_object_put(obj);
1704err_ctx_lo:
1705 kernel_context_close(ctx_lo);
1706err_ctx_hi:
1707 kernel_context_close(ctx_hi);
1708 return err;
1709}
1710
1711static struct i915_request *
1712spinner_create_request(struct igt_spinner *spin,
1713 struct i915_gem_context *ctx,
1714 struct intel_engine_cs *engine,
1715 u32 arb)
1716{
1717 struct intel_context *ce;
1718 struct i915_request *rq;
1719
1720 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
1721 if (IS_ERR(ce))
1722 return ERR_CAST(ce);
1723
1724 rq = igt_spinner_create_request(spin, ce, arb);
1725 intel_context_put(ce);
1726 return rq;
1727}
1728
1729static int live_preempt(void *arg)
1730{
1731 struct intel_gt *gt = arg;
1732 struct i915_gem_context *ctx_hi, *ctx_lo;
1733 struct igt_spinner spin_hi, spin_lo;
1734 struct intel_engine_cs *engine;
1735 enum intel_engine_id id;
1736 int err = -ENOMEM;
1737
1738 if (igt_spinner_init(&spin_hi, gt))
1739 return -ENOMEM;
1740
1741 if (igt_spinner_init(&spin_lo, gt))
1742 goto err_spin_hi;
1743
1744 ctx_hi = kernel_context(gt->i915);
1745 if (!ctx_hi)
1746 goto err_spin_lo;
1747 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
1748
1749 ctx_lo = kernel_context(gt->i915);
1750 if (!ctx_lo)
1751 goto err_ctx_hi;
1752 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
1753
1754 for_each_engine(engine, gt, id) {
1755 struct igt_live_test t;
1756 struct i915_request *rq;
1757
1758 if (!intel_engine_has_preemption(engine))
1759 continue;
1760
1761 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1762 err = -EIO;
1763 goto err_ctx_lo;
1764 }
1765
1766 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1767 MI_ARB_CHECK);
1768 if (IS_ERR(rq)) {
1769 err = PTR_ERR(rq);
1770 goto err_ctx_lo;
1771 }
1772
1773 i915_request_add(rq);
1774 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1775 GEM_TRACE("lo spinner failed to start\n");
1776 GEM_TRACE_DUMP();
1777 intel_gt_set_wedged(gt);
1778 err = -EIO;
1779 goto err_ctx_lo;
1780 }
1781
1782 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1783 MI_ARB_CHECK);
1784 if (IS_ERR(rq)) {
1785 igt_spinner_end(&spin_lo);
1786 err = PTR_ERR(rq);
1787 goto err_ctx_lo;
1788 }
1789
1790 i915_request_add(rq);
1791 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1792 GEM_TRACE("hi spinner failed to start\n");
1793 GEM_TRACE_DUMP();
1794 intel_gt_set_wedged(gt);
1795 err = -EIO;
1796 goto err_ctx_lo;
1797 }
1798
1799 igt_spinner_end(&spin_hi);
1800 igt_spinner_end(&spin_lo);
1801
1802 if (igt_live_test_end(&t)) {
1803 err = -EIO;
1804 goto err_ctx_lo;
1805 }
1806 }
1807
1808 err = 0;
1809err_ctx_lo:
1810 kernel_context_close(ctx_lo);
1811err_ctx_hi:
1812 kernel_context_close(ctx_hi);
1813err_spin_lo:
1814 igt_spinner_fini(&spin_lo);
1815err_spin_hi:
1816 igt_spinner_fini(&spin_hi);
1817 return err;
1818}
1819
1820static int live_late_preempt(void *arg)
1821{
1822 struct intel_gt *gt = arg;
1823 struct i915_gem_context *ctx_hi, *ctx_lo;
1824 struct igt_spinner spin_hi, spin_lo;
1825 struct intel_engine_cs *engine;
1826 struct i915_sched_attr attr = {};
1827 enum intel_engine_id id;
1828 int err = -ENOMEM;
1829
1830 if (igt_spinner_init(&spin_hi, gt))
1831 return -ENOMEM;
1832
1833 if (igt_spinner_init(&spin_lo, gt))
1834 goto err_spin_hi;
1835
1836 ctx_hi = kernel_context(gt->i915);
1837 if (!ctx_hi)
1838 goto err_spin_lo;
1839
1840 ctx_lo = kernel_context(gt->i915);
1841 if (!ctx_lo)
1842 goto err_ctx_hi;
1843
1844
1845 ctx_lo->sched.priority = 1;
1846
1847 for_each_engine(engine, gt, id) {
1848 struct igt_live_test t;
1849 struct i915_request *rq;
1850
1851 if (!intel_engine_has_preemption(engine))
1852 continue;
1853
1854 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
1855 err = -EIO;
1856 goto err_ctx_lo;
1857 }
1858
1859 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1860 MI_ARB_CHECK);
1861 if (IS_ERR(rq)) {
1862 err = PTR_ERR(rq);
1863 goto err_ctx_lo;
1864 }
1865
1866 i915_request_add(rq);
1867 if (!igt_wait_for_spinner(&spin_lo, rq)) {
1868 pr_err("First context failed to start\n");
1869 goto err_wedged;
1870 }
1871
1872 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1873 MI_NOOP);
1874 if (IS_ERR(rq)) {
1875 igt_spinner_end(&spin_lo);
1876 err = PTR_ERR(rq);
1877 goto err_ctx_lo;
1878 }
1879
1880 i915_request_add(rq);
1881 if (igt_wait_for_spinner(&spin_hi, rq)) {
1882 pr_err("Second context overtook first?\n");
1883 goto err_wedged;
1884 }
1885
1886 attr.priority = I915_PRIORITY_MAX;
1887 engine->schedule(rq, &attr);
1888
1889 if (!igt_wait_for_spinner(&spin_hi, rq)) {
1890 pr_err("High priority context failed to preempt the low priority context\n");
1891 GEM_TRACE_DUMP();
1892 goto err_wedged;
1893 }
1894
1895 igt_spinner_end(&spin_hi);
1896 igt_spinner_end(&spin_lo);
1897
1898 if (igt_live_test_end(&t)) {
1899 err = -EIO;
1900 goto err_ctx_lo;
1901 }
1902 }
1903
1904 err = 0;
1905err_ctx_lo:
1906 kernel_context_close(ctx_lo);
1907err_ctx_hi:
1908 kernel_context_close(ctx_hi);
1909err_spin_lo:
1910 igt_spinner_fini(&spin_lo);
1911err_spin_hi:
1912 igt_spinner_fini(&spin_hi);
1913 return err;
1914
1915err_wedged:
1916 igt_spinner_end(&spin_hi);
1917 igt_spinner_end(&spin_lo);
1918 intel_gt_set_wedged(gt);
1919 err = -EIO;
1920 goto err_ctx_lo;
1921}
1922
1923struct preempt_client {
1924 struct igt_spinner spin;
1925 struct i915_gem_context *ctx;
1926};
1927
1928static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
1929{
1930 c->ctx = kernel_context(gt->i915);
1931 if (!c->ctx)
1932 return -ENOMEM;
1933
1934 if (igt_spinner_init(&c->spin, gt))
1935 goto err_ctx;
1936
1937 return 0;
1938
1939err_ctx:
1940 kernel_context_close(c->ctx);
1941 return -ENOMEM;
1942}
1943
1944static void preempt_client_fini(struct preempt_client *c)
1945{
1946 igt_spinner_fini(&c->spin);
1947 kernel_context_close(c->ctx);
1948}
1949
1950static int live_nopreempt(void *arg)
1951{
1952 struct intel_gt *gt = arg;
1953 struct intel_engine_cs *engine;
1954 struct preempt_client a, b;
1955 enum intel_engine_id id;
1956 int err = -ENOMEM;
1957
1958
1959
1960
1961
1962
1963 if (preempt_client_init(gt, &a))
1964 return -ENOMEM;
1965 if (preempt_client_init(gt, &b))
1966 goto err_client_a;
1967 b.ctx->sched.priority = I915_PRIORITY_MAX;
1968
1969 for_each_engine(engine, gt, id) {
1970 struct i915_request *rq_a, *rq_b;
1971
1972 if (!intel_engine_has_preemption(engine))
1973 continue;
1974
1975 engine->execlists.preempt_hang.count = 0;
1976
1977 rq_a = spinner_create_request(&a.spin,
1978 a.ctx, engine,
1979 MI_ARB_CHECK);
1980 if (IS_ERR(rq_a)) {
1981 err = PTR_ERR(rq_a);
1982 goto err_client_b;
1983 }
1984
1985
1986 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
1987
1988 i915_request_add(rq_a);
1989 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1990 pr_err("First client failed to start\n");
1991 goto err_wedged;
1992 }
1993
1994 rq_b = spinner_create_request(&b.spin,
1995 b.ctx, engine,
1996 MI_ARB_CHECK);
1997 if (IS_ERR(rq_b)) {
1998 err = PTR_ERR(rq_b);
1999 goto err_client_b;
2000 }
2001
2002 i915_request_add(rq_b);
2003
2004
2005 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
2006
2007
2008 if (igt_wait_for_spinner(&b.spin, rq_b)) {
2009 pr_err("Second client started too early!\n");
2010 goto err_wedged;
2011 }
2012
2013 igt_spinner_end(&a.spin);
2014
2015 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2016 pr_err("Second client failed to start\n");
2017 goto err_wedged;
2018 }
2019
2020 igt_spinner_end(&b.spin);
2021
2022 if (engine->execlists.preempt_hang.count) {
2023 pr_err("Preemption recorded x%d; should have been suppressed!\n",
2024 engine->execlists.preempt_hang.count);
2025 err = -EINVAL;
2026 goto err_wedged;
2027 }
2028
2029 if (igt_flush_test(gt->i915))
2030 goto err_wedged;
2031 }
2032
2033 err = 0;
2034err_client_b:
2035 preempt_client_fini(&b);
2036err_client_a:
2037 preempt_client_fini(&a);
2038 return err;
2039
2040err_wedged:
2041 igt_spinner_end(&b.spin);
2042 igt_spinner_end(&a.spin);
2043 intel_gt_set_wedged(gt);
2044 err = -EIO;
2045 goto err_client_b;
2046}
2047
2048struct live_preempt_cancel {
2049 struct intel_engine_cs *engine;
2050 struct preempt_client a, b;
2051};
2052
2053static int __cancel_active0(struct live_preempt_cancel *arg)
2054{
2055 struct i915_request *rq;
2056 struct igt_live_test t;
2057 int err;
2058
2059
2060 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2061 if (igt_live_test_begin(&t, arg->engine->i915,
2062 __func__, arg->engine->name))
2063 return -EIO;
2064
2065 rq = spinner_create_request(&arg->a.spin,
2066 arg->a.ctx, arg->engine,
2067 MI_ARB_CHECK);
2068 if (IS_ERR(rq))
2069 return PTR_ERR(rq);
2070
2071 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2072 i915_request_get(rq);
2073 i915_request_add(rq);
2074 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2075 err = -EIO;
2076 goto out;
2077 }
2078
2079 intel_context_set_banned(rq->context);
2080 err = intel_engine_pulse(arg->engine);
2081 if (err)
2082 goto out;
2083
2084 err = wait_for_reset(arg->engine, rq, HZ / 2);
2085 if (err) {
2086 pr_err("Cancelled inflight0 request did not reset\n");
2087 goto out;
2088 }
2089
2090out:
2091 i915_request_put(rq);
2092 if (igt_live_test_end(&t))
2093 err = -EIO;
2094 return err;
2095}
2096
2097static int __cancel_active1(struct live_preempt_cancel *arg)
2098{
2099 struct i915_request *rq[2] = {};
2100 struct igt_live_test t;
2101 int err;
2102
2103
2104 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2105 if (igt_live_test_begin(&t, arg->engine->i915,
2106 __func__, arg->engine->name))
2107 return -EIO;
2108
2109 rq[0] = spinner_create_request(&arg->a.spin,
2110 arg->a.ctx, arg->engine,
2111 MI_NOOP);
2112 if (IS_ERR(rq[0]))
2113 return PTR_ERR(rq[0]);
2114
2115 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2116 i915_request_get(rq[0]);
2117 i915_request_add(rq[0]);
2118 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2119 err = -EIO;
2120 goto out;
2121 }
2122
2123 rq[1] = spinner_create_request(&arg->b.spin,
2124 arg->b.ctx, arg->engine,
2125 MI_ARB_CHECK);
2126 if (IS_ERR(rq[1])) {
2127 err = PTR_ERR(rq[1]);
2128 goto out;
2129 }
2130
2131 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2132 i915_request_get(rq[1]);
2133 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2134 i915_request_add(rq[1]);
2135 if (err)
2136 goto out;
2137
2138 intel_context_set_banned(rq[1]->context);
2139 err = intel_engine_pulse(arg->engine);
2140 if (err)
2141 goto out;
2142
2143 igt_spinner_end(&arg->a.spin);
2144 err = wait_for_reset(arg->engine, rq[1], HZ / 2);
2145 if (err)
2146 goto out;
2147
2148 if (rq[0]->fence.error != 0) {
2149 pr_err("Normal inflight0 request did not complete\n");
2150 err = -EINVAL;
2151 goto out;
2152 }
2153
2154 if (rq[1]->fence.error != -EIO) {
2155 pr_err("Cancelled inflight1 request did not report -EIO\n");
2156 err = -EINVAL;
2157 goto out;
2158 }
2159
2160out:
2161 i915_request_put(rq[1]);
2162 i915_request_put(rq[0]);
2163 if (igt_live_test_end(&t))
2164 err = -EIO;
2165 return err;
2166}
2167
2168static int __cancel_queued(struct live_preempt_cancel *arg)
2169{
2170 struct i915_request *rq[3] = {};
2171 struct igt_live_test t;
2172 int err;
2173
2174
2175 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2176 if (igt_live_test_begin(&t, arg->engine->i915,
2177 __func__, arg->engine->name))
2178 return -EIO;
2179
2180 rq[0] = spinner_create_request(&arg->a.spin,
2181 arg->a.ctx, arg->engine,
2182 MI_ARB_CHECK);
2183 if (IS_ERR(rq[0]))
2184 return PTR_ERR(rq[0]);
2185
2186 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
2187 i915_request_get(rq[0]);
2188 i915_request_add(rq[0]);
2189 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
2190 err = -EIO;
2191 goto out;
2192 }
2193
2194 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
2195 if (IS_ERR(rq[1])) {
2196 err = PTR_ERR(rq[1]);
2197 goto out;
2198 }
2199
2200 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
2201 i915_request_get(rq[1]);
2202 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
2203 i915_request_add(rq[1]);
2204 if (err)
2205 goto out;
2206
2207 rq[2] = spinner_create_request(&arg->b.spin,
2208 arg->a.ctx, arg->engine,
2209 MI_ARB_CHECK);
2210 if (IS_ERR(rq[2])) {
2211 err = PTR_ERR(rq[2]);
2212 goto out;
2213 }
2214
2215 i915_request_get(rq[2]);
2216 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
2217 i915_request_add(rq[2]);
2218 if (err)
2219 goto out;
2220
2221 intel_context_set_banned(rq[2]->context);
2222 err = intel_engine_pulse(arg->engine);
2223 if (err)
2224 goto out;
2225
2226 err = wait_for_reset(arg->engine, rq[2], HZ / 2);
2227 if (err)
2228 goto out;
2229
2230 if (rq[0]->fence.error != -EIO) {
2231 pr_err("Cancelled inflight0 request did not report -EIO\n");
2232 err = -EINVAL;
2233 goto out;
2234 }
2235
2236 if (rq[1]->fence.error != 0) {
2237 pr_err("Normal inflight1 request did not complete\n");
2238 err = -EINVAL;
2239 goto out;
2240 }
2241
2242 if (rq[2]->fence.error != -EIO) {
2243 pr_err("Cancelled queued request did not report -EIO\n");
2244 err = -EINVAL;
2245 goto out;
2246 }
2247
2248out:
2249 i915_request_put(rq[2]);
2250 i915_request_put(rq[1]);
2251 i915_request_put(rq[0]);
2252 if (igt_live_test_end(&t))
2253 err = -EIO;
2254 return err;
2255}
2256
2257static int __cancel_hostile(struct live_preempt_cancel *arg)
2258{
2259 struct i915_request *rq;
2260 int err;
2261
2262
2263 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
2264 return 0;
2265
2266 if (!intel_has_reset_engine(arg->engine->gt))
2267 return 0;
2268
2269 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
2270 rq = spinner_create_request(&arg->a.spin,
2271 arg->a.ctx, arg->engine,
2272 MI_NOOP);
2273 if (IS_ERR(rq))
2274 return PTR_ERR(rq);
2275
2276 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2277 i915_request_get(rq);
2278 i915_request_add(rq);
2279 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2280 err = -EIO;
2281 goto out;
2282 }
2283
2284 intel_context_set_banned(rq->context);
2285 err = intel_engine_pulse(arg->engine);
2286 if (err)
2287 goto out;
2288
2289 err = wait_for_reset(arg->engine, rq, HZ / 2);
2290 if (err) {
2291 pr_err("Cancelled inflight0 request did not reset\n");
2292 goto out;
2293 }
2294
2295out:
2296 i915_request_put(rq);
2297 if (igt_flush_test(arg->engine->i915))
2298 err = -EIO;
2299 return err;
2300}
2301
2302static void force_reset_timeout(struct intel_engine_cs *engine)
2303{
2304 engine->reset_timeout.probability = 999;
2305 atomic_set(&engine->reset_timeout.times, -1);
2306}
2307
2308static void cancel_reset_timeout(struct intel_engine_cs *engine)
2309{
2310 memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));
2311}
2312
2313static int __cancel_fail(struct live_preempt_cancel *arg)
2314{
2315 struct intel_engine_cs *engine = arg->engine;
2316 struct i915_request *rq;
2317 int err;
2318
2319 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
2320 return 0;
2321
2322 if (!intel_has_reset_engine(engine->gt))
2323 return 0;
2324
2325 GEM_TRACE("%s(%s)\n", __func__, engine->name);
2326 rq = spinner_create_request(&arg->a.spin,
2327 arg->a.ctx, engine,
2328 MI_NOOP);
2329 if (IS_ERR(rq))
2330 return PTR_ERR(rq);
2331
2332 clear_bit(CONTEXT_BANNED, &rq->context->flags);
2333 i915_request_get(rq);
2334 i915_request_add(rq);
2335 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2336 err = -EIO;
2337 goto out;
2338 }
2339
2340 intel_context_set_banned(rq->context);
2341
2342 err = intel_engine_pulse(engine);
2343 if (err)
2344 goto out;
2345
2346 force_reset_timeout(engine);
2347
2348
2349 while (!engine->execlists.pending[0])
2350 intel_engine_flush_submission(engine);
2351 del_timer_sync(&engine->execlists.preempt);
2352 intel_engine_flush_submission(engine);
2353
2354 cancel_reset_timeout(engine);
2355
2356
2357 intel_engine_set_heartbeat(engine, 1);
2358 err = wait_for_reset(engine, rq, HZ / 2);
2359 intel_engine_set_heartbeat(engine,
2360 engine->defaults.heartbeat_interval_ms);
2361 if (err) {
2362 pr_err("Cancelled inflight0 request did not reset\n");
2363 goto out;
2364 }
2365
2366out:
2367 i915_request_put(rq);
2368 if (igt_flush_test(engine->i915))
2369 err = -EIO;
2370 return err;
2371}
2372
2373static int live_preempt_cancel(void *arg)
2374{
2375 struct intel_gt *gt = arg;
2376 struct live_preempt_cancel data;
2377 enum intel_engine_id id;
2378 int err = -ENOMEM;
2379
2380
2381
2382
2383
2384
2385 if (preempt_client_init(gt, &data.a))
2386 return -ENOMEM;
2387 if (preempt_client_init(gt, &data.b))
2388 goto err_client_a;
2389
2390 for_each_engine(data.engine, gt, id) {
2391 if (!intel_engine_has_preemption(data.engine))
2392 continue;
2393
2394 err = __cancel_active0(&data);
2395 if (err)
2396 goto err_wedged;
2397
2398 err = __cancel_active1(&data);
2399 if (err)
2400 goto err_wedged;
2401
2402 err = __cancel_queued(&data);
2403 if (err)
2404 goto err_wedged;
2405
2406 err = __cancel_hostile(&data);
2407 if (err)
2408 goto err_wedged;
2409
2410 err = __cancel_fail(&data);
2411 if (err)
2412 goto err_wedged;
2413 }
2414
2415 err = 0;
2416err_client_b:
2417 preempt_client_fini(&data.b);
2418err_client_a:
2419 preempt_client_fini(&data.a);
2420 return err;
2421
2422err_wedged:
2423 GEM_TRACE_DUMP();
2424 igt_spinner_end(&data.b.spin);
2425 igt_spinner_end(&data.a.spin);
2426 intel_gt_set_wedged(gt);
2427 goto err_client_b;
2428}
2429
2430static int live_suppress_self_preempt(void *arg)
2431{
2432 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2433 struct intel_gt *gt = arg;
2434 struct intel_engine_cs *engine;
2435 struct preempt_client a, b;
2436 enum intel_engine_id id;
2437 int err = -ENOMEM;
2438
2439
2440
2441
2442
2443
2444
2445
2446 if (intel_uc_uses_guc_submission(>->uc))
2447 return 0;
2448
2449 if (intel_vgpu_active(gt->i915))
2450 return 0;
2451
2452 if (preempt_client_init(gt, &a))
2453 return -ENOMEM;
2454 if (preempt_client_init(gt, &b))
2455 goto err_client_a;
2456
2457 for_each_engine(engine, gt, id) {
2458 struct i915_request *rq_a, *rq_b;
2459 int depth;
2460
2461 if (!intel_engine_has_preemption(engine))
2462 continue;
2463
2464 if (igt_flush_test(gt->i915))
2465 goto err_wedged;
2466
2467 st_engine_heartbeat_disable(engine);
2468 engine->execlists.preempt_hang.count = 0;
2469
2470 rq_a = spinner_create_request(&a.spin,
2471 a.ctx, engine,
2472 MI_NOOP);
2473 if (IS_ERR(rq_a)) {
2474 err = PTR_ERR(rq_a);
2475 st_engine_heartbeat_enable(engine);
2476 goto err_client_b;
2477 }
2478
2479 i915_request_add(rq_a);
2480 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2481 pr_err("First client failed to start\n");
2482 st_engine_heartbeat_enable(engine);
2483 goto err_wedged;
2484 }
2485
2486
2487 mod_timer(&engine->execlists.timer, jiffies + HZ);
2488 for (depth = 0; depth < 8; depth++) {
2489 rq_b = spinner_create_request(&b.spin,
2490 b.ctx, engine,
2491 MI_NOOP);
2492 if (IS_ERR(rq_b)) {
2493 err = PTR_ERR(rq_b);
2494 st_engine_heartbeat_enable(engine);
2495 goto err_client_b;
2496 }
2497 i915_request_add(rq_b);
2498
2499 GEM_BUG_ON(i915_request_completed(rq_a));
2500 engine->schedule(rq_a, &attr);
2501 igt_spinner_end(&a.spin);
2502
2503 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2504 pr_err("Second client failed to start\n");
2505 st_engine_heartbeat_enable(engine);
2506 goto err_wedged;
2507 }
2508
2509 swap(a, b);
2510 rq_a = rq_b;
2511 }
2512 igt_spinner_end(&a.spin);
2513
2514 if (engine->execlists.preempt_hang.count) {
2515 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2516 engine->name,
2517 engine->execlists.preempt_hang.count,
2518 depth);
2519 st_engine_heartbeat_enable(engine);
2520 err = -EINVAL;
2521 goto err_client_b;
2522 }
2523
2524 st_engine_heartbeat_enable(engine);
2525 if (igt_flush_test(gt->i915))
2526 goto err_wedged;
2527 }
2528
2529 err = 0;
2530err_client_b:
2531 preempt_client_fini(&b);
2532err_client_a:
2533 preempt_client_fini(&a);
2534 return err;
2535
2536err_wedged:
2537 igt_spinner_end(&b.spin);
2538 igt_spinner_end(&a.spin);
2539 intel_gt_set_wedged(gt);
2540 err = -EIO;
2541 goto err_client_b;
2542}
2543
2544static int live_chain_preempt(void *arg)
2545{
2546 struct intel_gt *gt = arg;
2547 struct intel_engine_cs *engine;
2548 struct preempt_client hi, lo;
2549 enum intel_engine_id id;
2550 int err = -ENOMEM;
2551
2552
2553
2554
2555
2556
2557
2558 if (preempt_client_init(gt, &hi))
2559 return -ENOMEM;
2560
2561 if (preempt_client_init(gt, &lo))
2562 goto err_client_hi;
2563
2564 for_each_engine(engine, gt, id) {
2565 struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };
2566 struct igt_live_test t;
2567 struct i915_request *rq;
2568 int ring_size, count, i;
2569
2570 if (!intel_engine_has_preemption(engine))
2571 continue;
2572
2573 rq = spinner_create_request(&lo.spin,
2574 lo.ctx, engine,
2575 MI_ARB_CHECK);
2576 if (IS_ERR(rq))
2577 goto err_wedged;
2578
2579 i915_request_get(rq);
2580 i915_request_add(rq);
2581
2582 ring_size = rq->wa_tail - rq->head;
2583 if (ring_size < 0)
2584 ring_size += rq->ring->size;
2585 ring_size = rq->ring->size / ring_size;
2586 pr_debug("%s(%s): Using maximum of %d requests\n",
2587 __func__, engine->name, ring_size);
2588
2589 igt_spinner_end(&lo.spin);
2590 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
2591 pr_err("Timed out waiting to flush %s\n", engine->name);
2592 i915_request_put(rq);
2593 goto err_wedged;
2594 }
2595 i915_request_put(rq);
2596
2597 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
2598 err = -EIO;
2599 goto err_wedged;
2600 }
2601
2602 for_each_prime_number_from(count, 1, ring_size) {
2603 rq = spinner_create_request(&hi.spin,
2604 hi.ctx, engine,
2605 MI_ARB_CHECK);
2606 if (IS_ERR(rq))
2607 goto err_wedged;
2608 i915_request_add(rq);
2609 if (!igt_wait_for_spinner(&hi.spin, rq))
2610 goto err_wedged;
2611
2612 rq = spinner_create_request(&lo.spin,
2613 lo.ctx, engine,
2614 MI_ARB_CHECK);
2615 if (IS_ERR(rq))
2616 goto err_wedged;
2617 i915_request_add(rq);
2618
2619 for (i = 0; i < count; i++) {
2620 rq = igt_request_alloc(lo.ctx, engine);
2621 if (IS_ERR(rq))
2622 goto err_wedged;
2623 i915_request_add(rq);
2624 }
2625
2626 rq = igt_request_alloc(hi.ctx, engine);
2627 if (IS_ERR(rq))
2628 goto err_wedged;
2629
2630 i915_request_get(rq);
2631 i915_request_add(rq);
2632 engine->schedule(rq, &attr);
2633
2634 igt_spinner_end(&hi.spin);
2635 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2636 struct drm_printer p =
2637 drm_info_printer(gt->i915->drm.dev);
2638
2639 pr_err("Failed to preempt over chain of %d\n",
2640 count);
2641 intel_engine_dump(engine, &p,
2642 "%s\n", engine->name);
2643 i915_request_put(rq);
2644 goto err_wedged;
2645 }
2646 igt_spinner_end(&lo.spin);
2647 i915_request_put(rq);
2648
2649 rq = igt_request_alloc(lo.ctx, engine);
2650 if (IS_ERR(rq))
2651 goto err_wedged;
2652
2653 i915_request_get(rq);
2654 i915_request_add(rq);
2655
2656 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
2657 struct drm_printer p =
2658 drm_info_printer(gt->i915->drm.dev);
2659
2660 pr_err("Failed to flush low priority chain of %d requests\n",
2661 count);
2662 intel_engine_dump(engine, &p,
2663 "%s\n", engine->name);
2664
2665 i915_request_put(rq);
2666 goto err_wedged;
2667 }
2668 i915_request_put(rq);
2669 }
2670
2671 if (igt_live_test_end(&t)) {
2672 err = -EIO;
2673 goto err_wedged;
2674 }
2675 }
2676
2677 err = 0;
2678err_client_lo:
2679 preempt_client_fini(&lo);
2680err_client_hi:
2681 preempt_client_fini(&hi);
2682 return err;
2683
2684err_wedged:
2685 igt_spinner_end(&hi.spin);
2686 igt_spinner_end(&lo.spin);
2687 intel_gt_set_wedged(gt);
2688 err = -EIO;
2689 goto err_client_lo;
2690}
2691
2692static int create_gang(struct intel_engine_cs *engine,
2693 struct i915_request **prev)
2694{
2695 struct drm_i915_gem_object *obj;
2696 struct intel_context *ce;
2697 struct i915_request *rq;
2698 struct i915_vma *vma;
2699 u32 *cs;
2700 int err;
2701
2702 ce = intel_context_create(engine);
2703 if (IS_ERR(ce))
2704 return PTR_ERR(ce);
2705
2706 obj = i915_gem_object_create_internal(engine->i915, 4096);
2707 if (IS_ERR(obj)) {
2708 err = PTR_ERR(obj);
2709 goto err_ce;
2710 }
2711
2712 vma = i915_vma_instance(obj, ce->vm, NULL);
2713 if (IS_ERR(vma)) {
2714 err = PTR_ERR(vma);
2715 goto err_obj;
2716 }
2717
2718 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2719 if (err)
2720 goto err_obj;
2721
2722 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
2723 if (IS_ERR(cs)) {
2724 err = PTR_ERR(cs);
2725 goto err_obj;
2726 }
2727
2728
2729 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2730
2731 *cs++ = MI_SEMAPHORE_WAIT |
2732 MI_SEMAPHORE_POLL |
2733 MI_SEMAPHORE_SAD_EQ_SDD;
2734 *cs++ = 0;
2735 *cs++ = lower_32_bits(vma->node.start);
2736 *cs++ = upper_32_bits(vma->node.start);
2737
2738 if (*prev) {
2739 u64 offset = (*prev)->batch->node.start;
2740
2741
2742 *cs++ = MI_STORE_DWORD_IMM_GEN4;
2743 *cs++ = lower_32_bits(offset);
2744 *cs++ = upper_32_bits(offset);
2745 *cs++ = 0;
2746 }
2747
2748 *cs++ = MI_BATCH_BUFFER_END;
2749 i915_gem_object_flush_map(obj);
2750 i915_gem_object_unpin_map(obj);
2751
2752 rq = intel_context_create_request(ce);
2753 if (IS_ERR(rq)) {
2754 err = PTR_ERR(rq);
2755 goto err_obj;
2756 }
2757
2758 rq->batch = i915_vma_get(vma);
2759 i915_request_get(rq);
2760
2761 i915_vma_lock(vma);
2762 err = i915_request_await_object(rq, vma->obj, false);
2763 if (!err)
2764 err = i915_vma_move_to_active(vma, rq, 0);
2765 if (!err)
2766 err = rq->engine->emit_bb_start(rq,
2767 vma->node.start,
2768 PAGE_SIZE, 0);
2769 i915_vma_unlock(vma);
2770 i915_request_add(rq);
2771 if (err)
2772 goto err_rq;
2773
2774 i915_gem_object_put(obj);
2775 intel_context_put(ce);
2776
2777 rq->mock.link.next = &(*prev)->mock.link;
2778 *prev = rq;
2779 return 0;
2780
2781err_rq:
2782 i915_vma_put(rq->batch);
2783 i915_request_put(rq);
2784err_obj:
2785 i915_gem_object_put(obj);
2786err_ce:
2787 intel_context_put(ce);
2788 return err;
2789}
2790
2791static int __live_preempt_ring(struct intel_engine_cs *engine,
2792 struct igt_spinner *spin,
2793 int queue_sz, int ring_sz)
2794{
2795 struct intel_context *ce[2] = {};
2796 struct i915_request *rq;
2797 struct igt_live_test t;
2798 int err = 0;
2799 int n;
2800
2801 if (igt_live_test_begin(&t, engine->i915, __func__, engine->name))
2802 return -EIO;
2803
2804 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2805 struct intel_context *tmp;
2806
2807 tmp = intel_context_create(engine);
2808 if (IS_ERR(tmp)) {
2809 err = PTR_ERR(tmp);
2810 goto err_ce;
2811 }
2812
2813 tmp->ring = __intel_context_ring_size(ring_sz);
2814
2815 err = intel_context_pin(tmp);
2816 if (err) {
2817 intel_context_put(tmp);
2818 goto err_ce;
2819 }
2820
2821 memset32(tmp->ring->vaddr,
2822 0xdeadbeef,
2823 tmp->ring->vma->size / sizeof(u32));
2824
2825 ce[n] = tmp;
2826 }
2827
2828 rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK);
2829 if (IS_ERR(rq)) {
2830 err = PTR_ERR(rq);
2831 goto err_ce;
2832 }
2833
2834 i915_request_get(rq);
2835 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2836 i915_request_add(rq);
2837
2838 if (!igt_wait_for_spinner(spin, rq)) {
2839 intel_gt_set_wedged(engine->gt);
2840 i915_request_put(rq);
2841 err = -ETIME;
2842 goto err_ce;
2843 }
2844
2845
2846 n = 0;
2847 while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) {
2848 struct i915_request *tmp;
2849
2850 tmp = intel_context_create_request(ce[0]);
2851 if (IS_ERR(tmp)) {
2852 err = PTR_ERR(tmp);
2853 i915_request_put(rq);
2854 goto err_ce;
2855 }
2856
2857 i915_request_add(tmp);
2858 intel_engine_flush_submission(engine);
2859 n++;
2860 }
2861 intel_engine_flush_submission(engine);
2862 pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",
2863 engine->name, queue_sz, n,
2864 ce[0]->ring->size,
2865 ce[0]->ring->tail,
2866 ce[0]->ring->emit,
2867 rq->tail);
2868 i915_request_put(rq);
2869
2870
2871 rq = intel_context_create_request(ce[1]);
2872 if (IS_ERR(rq)) {
2873 err = PTR_ERR(rq);
2874 goto err_ce;
2875 }
2876
2877 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2878 i915_request_get(rq);
2879 i915_request_add(rq);
2880
2881 err = wait_for_submit(engine, rq, HZ / 2);
2882 i915_request_put(rq);
2883 if (err) {
2884 pr_err("%s: preemption request was not submitted\n",
2885 engine->name);
2886 err = -ETIME;
2887 }
2888
2889 pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",
2890 engine->name,
2891 ce[0]->ring->tail, ce[0]->ring->emit,
2892 ce[1]->ring->tail, ce[1]->ring->emit);
2893
2894err_ce:
2895 intel_engine_flush_submission(engine);
2896 igt_spinner_end(spin);
2897 for (n = 0; n < ARRAY_SIZE(ce); n++) {
2898 if (IS_ERR_OR_NULL(ce[n]))
2899 break;
2900
2901 intel_context_unpin(ce[n]);
2902 intel_context_put(ce[n]);
2903 }
2904 if (igt_live_test_end(&t))
2905 err = -EIO;
2906 return err;
2907}
2908
2909static int live_preempt_ring(void *arg)
2910{
2911 struct intel_gt *gt = arg;
2912 struct intel_engine_cs *engine;
2913 struct igt_spinner spin;
2914 enum intel_engine_id id;
2915 int err = 0;
2916
2917
2918
2919
2920
2921
2922
2923 if (igt_spinner_init(&spin, gt))
2924 return -ENOMEM;
2925
2926 for_each_engine(engine, gt, id) {
2927 int n;
2928
2929 if (!intel_engine_has_preemption(engine))
2930 continue;
2931
2932 if (!intel_engine_can_store_dword(engine))
2933 continue;
2934
2935 st_engine_heartbeat_disable(engine);
2936
2937 for (n = 0; n <= 3; n++) {
2938 err = __live_preempt_ring(engine, &spin,
2939 n * SZ_4K / 4, SZ_4K);
2940 if (err)
2941 break;
2942 }
2943
2944 st_engine_heartbeat_enable(engine);
2945 if (err)
2946 break;
2947 }
2948
2949 igt_spinner_fini(&spin);
2950 return err;
2951}
2952
2953static int live_preempt_gang(void *arg)
2954{
2955 struct intel_gt *gt = arg;
2956 struct intel_engine_cs *engine;
2957 enum intel_engine_id id;
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969 for_each_engine(engine, gt, id) {
2970 struct i915_request *rq = NULL;
2971 struct igt_live_test t;
2972 IGT_TIMEOUT(end_time);
2973 int prio = 0;
2974 int err = 0;
2975 u32 *cs;
2976
2977 if (!intel_engine_has_preemption(engine))
2978 continue;
2979
2980 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2981 return -EIO;
2982
2983 do {
2984 struct i915_sched_attr attr = { .priority = prio++ };
2985
2986 err = create_gang(engine, &rq);
2987 if (err)
2988 break;
2989
2990
2991 engine->schedule(rq, &attr);
2992 } while (prio <= I915_PRIORITY_MAX &&
2993 !__igt_timeout(end_time, NULL));
2994 pr_debug("%s: Preempt chain of %d requests\n",
2995 engine->name, prio);
2996
2997
2998
2999
3000
3001
3002
3003 cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);
3004 if (!IS_ERR(cs)) {
3005 *cs = 0;
3006 i915_gem_object_unpin_map(rq->batch->obj);
3007 } else {
3008 err = PTR_ERR(cs);
3009 intel_gt_set_wedged(gt);
3010 }
3011
3012 while (rq) {
3013 struct i915_request *n = list_next_entry(rq, mock.link);
3014
3015 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
3016 struct drm_printer p =
3017 drm_info_printer(engine->i915->drm.dev);
3018
3019 pr_err("Failed to flush chain of %d requests, at %d\n",
3020 prio, rq_prio(rq));
3021 intel_engine_dump(engine, &p,
3022 "%s\n", engine->name);
3023
3024 err = -ETIME;
3025 }
3026
3027 i915_vma_put(rq->batch);
3028 i915_request_put(rq);
3029 rq = n;
3030 }
3031
3032 if (igt_live_test_end(&t))
3033 err = -EIO;
3034 if (err)
3035 return err;
3036 }
3037
3038 return 0;
3039}
3040
3041static struct i915_vma *
3042create_gpr_user(struct intel_engine_cs *engine,
3043 struct i915_vma *result,
3044 unsigned int offset)
3045{
3046 struct drm_i915_gem_object *obj;
3047 struct i915_vma *vma;
3048 u32 *cs;
3049 int err;
3050 int i;
3051
3052 obj = i915_gem_object_create_internal(engine->i915, 4096);
3053 if (IS_ERR(obj))
3054 return ERR_CAST(obj);
3055
3056 vma = i915_vma_instance(obj, result->vm, NULL);
3057 if (IS_ERR(vma)) {
3058 i915_gem_object_put(obj);
3059 return vma;
3060 }
3061
3062 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3063 if (err) {
3064 i915_vma_put(vma);
3065 return ERR_PTR(err);
3066 }
3067
3068 cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
3069 if (IS_ERR(cs)) {
3070 i915_vma_put(vma);
3071 return ERR_CAST(cs);
3072 }
3073
3074
3075 *cs++ = MI_LOAD_REGISTER_IMM(1);
3076 *cs++ = CS_GPR(engine, 0);
3077 *cs++ = 1;
3078
3079 for (i = 1; i < NUM_GPR; i++) {
3080 u64 addr;
3081
3082
3083
3084
3085
3086
3087
3088
3089 *cs++ = MI_MATH(4);
3090 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));
3091 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));
3092 *cs++ = MI_MATH_ADD;
3093 *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);
3094
3095 addr = result->node.start + offset + i * sizeof(*cs);
3096 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
3097 *cs++ = CS_GPR(engine, 2 * i);
3098 *cs++ = lower_32_bits(addr);
3099 *cs++ = upper_32_bits(addr);
3100
3101 *cs++ = MI_SEMAPHORE_WAIT |
3102 MI_SEMAPHORE_POLL |
3103 MI_SEMAPHORE_SAD_GTE_SDD;
3104 *cs++ = i;
3105 *cs++ = lower_32_bits(result->node.start);
3106 *cs++ = upper_32_bits(result->node.start);
3107 }
3108
3109 *cs++ = MI_BATCH_BUFFER_END;
3110 i915_gem_object_flush_map(obj);
3111 i915_gem_object_unpin_map(obj);
3112
3113 return vma;
3114}
3115
3116static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)
3117{
3118 struct drm_i915_gem_object *obj;
3119 struct i915_vma *vma;
3120 int err;
3121
3122 obj = i915_gem_object_create_internal(gt->i915, sz);
3123 if (IS_ERR(obj))
3124 return ERR_CAST(obj);
3125
3126 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
3127 if (IS_ERR(vma)) {
3128 i915_gem_object_put(obj);
3129 return vma;
3130 }
3131
3132 err = i915_ggtt_pin(vma, NULL, 0, 0);
3133 if (err) {
3134 i915_vma_put(vma);
3135 return ERR_PTR(err);
3136 }
3137
3138 return vma;
3139}
3140
3141static struct i915_request *
3142create_gpr_client(struct intel_engine_cs *engine,
3143 struct i915_vma *global,
3144 unsigned int offset)
3145{
3146 struct i915_vma *batch, *vma;
3147 struct intel_context *ce;
3148 struct i915_request *rq;
3149 int err;
3150
3151 ce = intel_context_create(engine);
3152 if (IS_ERR(ce))
3153 return ERR_CAST(ce);
3154
3155 vma = i915_vma_instance(global->obj, ce->vm, NULL);
3156 if (IS_ERR(vma)) {
3157 err = PTR_ERR(vma);
3158 goto out_ce;
3159 }
3160
3161 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3162 if (err)
3163 goto out_ce;
3164
3165 batch = create_gpr_user(engine, vma, offset);
3166 if (IS_ERR(batch)) {
3167 err = PTR_ERR(batch);
3168 goto out_vma;
3169 }
3170
3171 rq = intel_context_create_request(ce);
3172 if (IS_ERR(rq)) {
3173 err = PTR_ERR(rq);
3174 goto out_batch;
3175 }
3176
3177 i915_vma_lock(vma);
3178 err = i915_request_await_object(rq, vma->obj, false);
3179 if (!err)
3180 err = i915_vma_move_to_active(vma, rq, 0);
3181 i915_vma_unlock(vma);
3182
3183 i915_vma_lock(batch);
3184 if (!err)
3185 err = i915_request_await_object(rq, batch->obj, false);
3186 if (!err)
3187 err = i915_vma_move_to_active(batch, rq, 0);
3188 if (!err)
3189 err = rq->engine->emit_bb_start(rq,
3190 batch->node.start,
3191 PAGE_SIZE, 0);
3192 i915_vma_unlock(batch);
3193 i915_vma_unpin(batch);
3194
3195 if (!err)
3196 i915_request_get(rq);
3197 i915_request_add(rq);
3198
3199out_batch:
3200 i915_vma_put(batch);
3201out_vma:
3202 i915_vma_unpin(vma);
3203out_ce:
3204 intel_context_put(ce);
3205 return err ? ERR_PTR(err) : rq;
3206}
3207
3208static int preempt_user(struct intel_engine_cs *engine,
3209 struct i915_vma *global,
3210 int id)
3211{
3212 struct i915_sched_attr attr = {
3213 .priority = I915_PRIORITY_MAX
3214 };
3215 struct i915_request *rq;
3216 int err = 0;
3217 u32 *cs;
3218
3219 rq = intel_engine_create_kernel_request(engine);
3220 if (IS_ERR(rq))
3221 return PTR_ERR(rq);
3222
3223 cs = intel_ring_begin(rq, 4);
3224 if (IS_ERR(cs)) {
3225 i915_request_add(rq);
3226 return PTR_ERR(cs);
3227 }
3228
3229 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
3230 *cs++ = i915_ggtt_offset(global);
3231 *cs++ = 0;
3232 *cs++ = id;
3233
3234 intel_ring_advance(rq, cs);
3235
3236 i915_request_get(rq);
3237 i915_request_add(rq);
3238
3239 engine->schedule(rq, &attr);
3240
3241 if (i915_request_wait(rq, 0, HZ / 2) < 0)
3242 err = -ETIME;
3243 i915_request_put(rq);
3244
3245 return err;
3246}
3247
3248static int live_preempt_user(void *arg)
3249{
3250 struct intel_gt *gt = arg;
3251 struct intel_engine_cs *engine;
3252 struct i915_vma *global;
3253 enum intel_engine_id id;
3254 u32 *result;
3255 int err = 0;
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271 global = create_global(gt, 4096);
3272 if (IS_ERR(global))
3273 return PTR_ERR(global);
3274
3275 result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);
3276 if (IS_ERR(result)) {
3277 i915_vma_unpin_and_release(&global, 0);
3278 return PTR_ERR(result);
3279 }
3280
3281 for_each_engine(engine, gt, id) {
3282 struct i915_request *client[3] = {};
3283 struct igt_live_test t;
3284 int i;
3285
3286 if (!intel_engine_has_preemption(engine))
3287 continue;
3288
3289 if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS)
3290 continue;
3291
3292 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
3293 err = -EIO;
3294 break;
3295 }
3296
3297 memset(result, 0, 4096);
3298
3299 for (i = 0; i < ARRAY_SIZE(client); i++) {
3300 struct i915_request *rq;
3301
3302 rq = create_gpr_client(engine, global,
3303 NUM_GPR * i * sizeof(u32));
3304 if (IS_ERR(rq)) {
3305 err = PTR_ERR(rq);
3306 goto end_test;
3307 }
3308
3309 client[i] = rq;
3310 }
3311
3312
3313 for (i = 1; i <= NUM_GPR; i++) {
3314 err = preempt_user(engine, global, i);
3315 if (err)
3316 goto end_test;
3317 }
3318
3319 if (READ_ONCE(result[0]) != NUM_GPR) {
3320 pr_err("%s: Failed to release semaphore\n",
3321 engine->name);
3322 err = -EIO;
3323 goto end_test;
3324 }
3325
3326 for (i = 0; i < ARRAY_SIZE(client); i++) {
3327 int gpr;
3328
3329 if (i915_request_wait(client[i], 0, HZ / 2) < 0) {
3330 err = -ETIME;
3331 goto end_test;
3332 }
3333
3334 for (gpr = 1; gpr < NUM_GPR; gpr++) {
3335 if (result[NUM_GPR * i + gpr] != 1) {
3336 pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",
3337 engine->name,
3338 i, gpr, result[NUM_GPR * i + gpr]);
3339 err = -EINVAL;
3340 goto end_test;
3341 }
3342 }
3343 }
3344
3345end_test:
3346 for (i = 0; i < ARRAY_SIZE(client); i++) {
3347 if (!client[i])
3348 break;
3349
3350 i915_request_put(client[i]);
3351 }
3352
3353
3354 smp_store_mb(result[0], -1);
3355 if (igt_live_test_end(&t))
3356 err = -EIO;
3357 if (err)
3358 break;
3359 }
3360
3361 i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);
3362 return err;
3363}
3364
3365static int live_preempt_timeout(void *arg)
3366{
3367 struct intel_gt *gt = arg;
3368 struct i915_gem_context *ctx_hi, *ctx_lo;
3369 struct igt_spinner spin_lo;
3370 struct intel_engine_cs *engine;
3371 enum intel_engine_id id;
3372 int err = -ENOMEM;
3373
3374
3375
3376
3377
3378 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
3379 return 0;
3380
3381 if (!intel_has_reset_engine(gt))
3382 return 0;
3383
3384 if (igt_spinner_init(&spin_lo, gt))
3385 return -ENOMEM;
3386
3387 ctx_hi = kernel_context(gt->i915);
3388 if (!ctx_hi)
3389 goto err_spin_lo;
3390 ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;
3391
3392 ctx_lo = kernel_context(gt->i915);
3393 if (!ctx_lo)
3394 goto err_ctx_hi;
3395 ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;
3396
3397 for_each_engine(engine, gt, id) {
3398 unsigned long saved_timeout;
3399 struct i915_request *rq;
3400
3401 if (!intel_engine_has_preemption(engine))
3402 continue;
3403
3404 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
3405 MI_NOOP);
3406 if (IS_ERR(rq)) {
3407 err = PTR_ERR(rq);
3408 goto err_ctx_lo;
3409 }
3410
3411 i915_request_add(rq);
3412 if (!igt_wait_for_spinner(&spin_lo, rq)) {
3413 intel_gt_set_wedged(gt);
3414 err = -EIO;
3415 goto err_ctx_lo;
3416 }
3417
3418 rq = igt_request_alloc(ctx_hi, engine);
3419 if (IS_ERR(rq)) {
3420 igt_spinner_end(&spin_lo);
3421 err = PTR_ERR(rq);
3422 goto err_ctx_lo;
3423 }
3424
3425
3426 while (READ_ONCE(engine->execlists.pending[0]))
3427 cpu_relax();
3428
3429 saved_timeout = engine->props.preempt_timeout_ms;
3430 engine->props.preempt_timeout_ms = 1;
3431
3432 i915_request_get(rq);
3433 i915_request_add(rq);
3434
3435 intel_engine_flush_submission(engine);
3436 engine->props.preempt_timeout_ms = saved_timeout;
3437
3438 if (i915_request_wait(rq, 0, HZ / 10) < 0) {
3439 intel_gt_set_wedged(gt);
3440 i915_request_put(rq);
3441 err = -ETIME;
3442 goto err_ctx_lo;
3443 }
3444
3445 igt_spinner_end(&spin_lo);
3446 i915_request_put(rq);
3447 }
3448
3449 err = 0;
3450err_ctx_lo:
3451 kernel_context_close(ctx_lo);
3452err_ctx_hi:
3453 kernel_context_close(ctx_hi);
3454err_spin_lo:
3455 igt_spinner_fini(&spin_lo);
3456 return err;
3457}
3458
3459static int random_range(struct rnd_state *rnd, int min, int max)
3460{
3461 return i915_prandom_u32_max_state(max - min, rnd) + min;
3462}
3463
3464static int random_priority(struct rnd_state *rnd)
3465{
3466 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
3467}
3468
3469struct preempt_smoke {
3470 struct intel_gt *gt;
3471 struct i915_gem_context **contexts;
3472 struct intel_engine_cs *engine;
3473 struct drm_i915_gem_object *batch;
3474 unsigned int ncontext;
3475 struct rnd_state prng;
3476 unsigned long count;
3477};
3478
3479static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
3480{
3481 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
3482 &smoke->prng)];
3483}
3484
3485static int smoke_submit(struct preempt_smoke *smoke,
3486 struct i915_gem_context *ctx, int prio,
3487 struct drm_i915_gem_object *batch)
3488{
3489 struct i915_request *rq;
3490 struct i915_vma *vma = NULL;
3491 int err = 0;
3492
3493 if (batch) {
3494 struct i915_address_space *vm;
3495
3496 vm = i915_gem_context_get_vm_rcu(ctx);
3497 vma = i915_vma_instance(batch, vm, NULL);
3498 i915_vm_put(vm);
3499 if (IS_ERR(vma))
3500 return PTR_ERR(vma);
3501
3502 err = i915_vma_pin(vma, 0, 0, PIN_USER);
3503 if (err)
3504 return err;
3505 }
3506
3507 ctx->sched.priority = prio;
3508
3509 rq = igt_request_alloc(ctx, smoke->engine);
3510 if (IS_ERR(rq)) {
3511 err = PTR_ERR(rq);
3512 goto unpin;
3513 }
3514
3515 if (vma) {
3516 i915_vma_lock(vma);
3517 err = i915_request_await_object(rq, vma->obj, false);
3518 if (!err)
3519 err = i915_vma_move_to_active(vma, rq, 0);
3520 if (!err)
3521 err = rq->engine->emit_bb_start(rq,
3522 vma->node.start,
3523 PAGE_SIZE, 0);
3524 i915_vma_unlock(vma);
3525 }
3526
3527 i915_request_add(rq);
3528
3529unpin:
3530 if (vma)
3531 i915_vma_unpin(vma);
3532
3533 return err;
3534}
3535
3536static int smoke_crescendo_thread(void *arg)
3537{
3538 struct preempt_smoke *smoke = arg;
3539 IGT_TIMEOUT(end_time);
3540 unsigned long count;
3541
3542 count = 0;
3543 do {
3544 struct i915_gem_context *ctx = smoke_context(smoke);
3545 int err;
3546
3547 err = smoke_submit(smoke,
3548 ctx, count % I915_PRIORITY_MAX,
3549 smoke->batch);
3550 if (err)
3551 return err;
3552
3553 count++;
3554 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3555
3556 smoke->count = count;
3557 return 0;
3558}
3559
3560static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
3561#define BATCH BIT(0)
3562{
3563 struct task_struct *tsk[I915_NUM_ENGINES] = {};
3564 struct preempt_smoke arg[I915_NUM_ENGINES];
3565 struct intel_engine_cs *engine;
3566 enum intel_engine_id id;
3567 unsigned long count;
3568 int err = 0;
3569
3570 for_each_engine(engine, smoke->gt, id) {
3571 arg[id] = *smoke;
3572 arg[id].engine = engine;
3573 if (!(flags & BATCH))
3574 arg[id].batch = NULL;
3575 arg[id].count = 0;
3576
3577 tsk[id] = kthread_run(smoke_crescendo_thread, &arg,
3578 "igt/smoke:%d", id);
3579 if (IS_ERR(tsk[id])) {
3580 err = PTR_ERR(tsk[id]);
3581 break;
3582 }
3583 get_task_struct(tsk[id]);
3584 }
3585
3586 yield();
3587
3588 count = 0;
3589 for_each_engine(engine, smoke->gt, id) {
3590 int status;
3591
3592 if (IS_ERR_OR_NULL(tsk[id]))
3593 continue;
3594
3595 status = kthread_stop(tsk[id]);
3596 if (status && !err)
3597 err = status;
3598
3599 count += arg[id].count;
3600
3601 put_task_struct(tsk[id]);
3602 }
3603
3604 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3605 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3606 return 0;
3607}
3608
3609static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
3610{
3611 enum intel_engine_id id;
3612 IGT_TIMEOUT(end_time);
3613 unsigned long count;
3614
3615 count = 0;
3616 do {
3617 for_each_engine(smoke->engine, smoke->gt, id) {
3618 struct i915_gem_context *ctx = smoke_context(smoke);
3619 int err;
3620
3621 err = smoke_submit(smoke,
3622 ctx, random_priority(&smoke->prng),
3623 flags & BATCH ? smoke->batch : NULL);
3624 if (err)
3625 return err;
3626
3627 count++;
3628 }
3629 } while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));
3630
3631 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3632 count, flags, smoke->gt->info.num_engines, smoke->ncontext);
3633 return 0;
3634}
3635
3636static int live_preempt_smoke(void *arg)
3637{
3638 struct preempt_smoke smoke = {
3639 .gt = arg,
3640 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3641 .ncontext = 256,
3642 };
3643 const unsigned int phase[] = { 0, BATCH };
3644 struct igt_live_test t;
3645 int err = -ENOMEM;
3646 u32 *cs;
3647 int n;
3648
3649 smoke.contexts = kmalloc_array(smoke.ncontext,
3650 sizeof(*smoke.contexts),
3651 GFP_KERNEL);
3652 if (!smoke.contexts)
3653 return -ENOMEM;
3654
3655 smoke.batch =
3656 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
3657 if (IS_ERR(smoke.batch)) {
3658 err = PTR_ERR(smoke.batch);
3659 goto err_free;
3660 }
3661
3662 cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);
3663 if (IS_ERR(cs)) {
3664 err = PTR_ERR(cs);
3665 goto err_batch;
3666 }
3667 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3668 cs[n] = MI_ARB_CHECK;
3669 cs[n] = MI_BATCH_BUFFER_END;
3670 i915_gem_object_flush_map(smoke.batch);
3671 i915_gem_object_unpin_map(smoke.batch);
3672
3673 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
3674 err = -EIO;
3675 goto err_batch;
3676 }
3677
3678 for (n = 0; n < smoke.ncontext; n++) {
3679 smoke.contexts[n] = kernel_context(smoke.gt->i915);
3680 if (!smoke.contexts[n])
3681 goto err_ctx;
3682 }
3683
3684 for (n = 0; n < ARRAY_SIZE(phase); n++) {
3685 err = smoke_crescendo(&smoke, phase[n]);
3686 if (err)
3687 goto err_ctx;
3688
3689 err = smoke_random(&smoke, phase[n]);
3690 if (err)
3691 goto err_ctx;
3692 }
3693
3694err_ctx:
3695 if (igt_live_test_end(&t))
3696 err = -EIO;
3697
3698 for (n = 0; n < smoke.ncontext; n++) {
3699 if (!smoke.contexts[n])
3700 break;
3701 kernel_context_close(smoke.contexts[n]);
3702 }
3703
3704err_batch:
3705 i915_gem_object_put(smoke.batch);
3706err_free:
3707 kfree(smoke.contexts);
3708
3709 return err;
3710}
3711
3712static int nop_virtual_engine(struct intel_gt *gt,
3713 struct intel_engine_cs **siblings,
3714 unsigned int nsibling,
3715 unsigned int nctx,
3716 unsigned int flags)
3717#define CHAIN BIT(0)
3718{
3719 IGT_TIMEOUT(end_time);
3720 struct i915_request *request[16] = {};
3721 struct intel_context *ve[16];
3722 unsigned long n, prime, nc;
3723 struct igt_live_test t;
3724 ktime_t times[2] = {};
3725 int err;
3726
3727 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
3728
3729 for (n = 0; n < nctx; n++) {
3730 ve[n] = intel_execlists_create_virtual(siblings, nsibling);
3731 if (IS_ERR(ve[n])) {
3732 err = PTR_ERR(ve[n]);
3733 nctx = n;
3734 goto out;
3735 }
3736
3737 err = intel_context_pin(ve[n]);
3738 if (err) {
3739 intel_context_put(ve[n]);
3740 nctx = n;
3741 goto out;
3742 }
3743 }
3744
3745 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
3746 if (err)
3747 goto out;
3748
3749 for_each_prime_number_from(prime, 1, 8192) {
3750 times[1] = ktime_get_raw();
3751
3752 if (flags & CHAIN) {
3753 for (nc = 0; nc < nctx; nc++) {
3754 for (n = 0; n < prime; n++) {
3755 struct i915_request *rq;
3756
3757 rq = i915_request_create(ve[nc]);
3758 if (IS_ERR(rq)) {
3759 err = PTR_ERR(rq);
3760 goto out;
3761 }
3762
3763 if (request[nc])
3764 i915_request_put(request[nc]);
3765 request[nc] = i915_request_get(rq);
3766 i915_request_add(rq);
3767 }
3768 }
3769 } else {
3770 for (n = 0; n < prime; n++) {
3771 for (nc = 0; nc < nctx; nc++) {
3772 struct i915_request *rq;
3773
3774 rq = i915_request_create(ve[nc]);
3775 if (IS_ERR(rq)) {
3776 err = PTR_ERR(rq);
3777 goto out;
3778 }
3779
3780 if (request[nc])
3781 i915_request_put(request[nc]);
3782 request[nc] = i915_request_get(rq);
3783 i915_request_add(rq);
3784 }
3785 }
3786 }
3787
3788 for (nc = 0; nc < nctx; nc++) {
3789 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
3790 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3791 __func__, ve[0]->engine->name,
3792 request[nc]->fence.context,
3793 request[nc]->fence.seqno);
3794
3795 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3796 __func__, ve[0]->engine->name,
3797 request[nc]->fence.context,
3798 request[nc]->fence.seqno);
3799 GEM_TRACE_DUMP();
3800 intel_gt_set_wedged(gt);
3801 break;
3802 }
3803 }
3804
3805 times[1] = ktime_sub(ktime_get_raw(), times[1]);
3806 if (prime == 1)
3807 times[0] = times[1];
3808
3809 for (nc = 0; nc < nctx; nc++) {
3810 i915_request_put(request[nc]);
3811 request[nc] = NULL;
3812 }
3813
3814 if (__igt_timeout(end_time, NULL))
3815 break;
3816 }
3817
3818 err = igt_live_test_end(&t);
3819 if (err)
3820 goto out;
3821
3822 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3823 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3824 prime, div64_u64(ktime_to_ns(times[1]), prime));
3825
3826out:
3827 if (igt_flush_test(gt->i915))
3828 err = -EIO;
3829
3830 for (nc = 0; nc < nctx; nc++) {
3831 i915_request_put(request[nc]);
3832 intel_context_unpin(ve[nc]);
3833 intel_context_put(ve[nc]);
3834 }
3835 return err;
3836}
3837
3838static unsigned int
3839__select_siblings(struct intel_gt *gt,
3840 unsigned int class,
3841 struct intel_engine_cs **siblings,
3842 bool (*filter)(const struct intel_engine_cs *))
3843{
3844 unsigned int n = 0;
3845 unsigned int inst;
3846
3847 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3848 if (!gt->engine_class[class][inst])
3849 continue;
3850
3851 if (filter && !filter(gt->engine_class[class][inst]))
3852 continue;
3853
3854 siblings[n++] = gt->engine_class[class][inst];
3855 }
3856
3857 return n;
3858}
3859
3860static unsigned int
3861select_siblings(struct intel_gt *gt,
3862 unsigned int class,
3863 struct intel_engine_cs **siblings)
3864{
3865 return __select_siblings(gt, class, siblings, NULL);
3866}
3867
3868static int live_virtual_engine(void *arg)
3869{
3870 struct intel_gt *gt = arg;
3871 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3872 struct intel_engine_cs *engine;
3873 enum intel_engine_id id;
3874 unsigned int class;
3875 int err;
3876
3877 if (intel_uc_uses_guc_submission(>->uc))
3878 return 0;
3879
3880 for_each_engine(engine, gt, id) {
3881 err = nop_virtual_engine(gt, &engine, 1, 1, 0);
3882 if (err) {
3883 pr_err("Failed to wrap engine %s: err=%d\n",
3884 engine->name, err);
3885 return err;
3886 }
3887 }
3888
3889 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3890 int nsibling, n;
3891
3892 nsibling = select_siblings(gt, class, siblings);
3893 if (nsibling < 2)
3894 continue;
3895
3896 for (n = 1; n <= nsibling + 1; n++) {
3897 err = nop_virtual_engine(gt, siblings, nsibling,
3898 n, 0);
3899 if (err)
3900 return err;
3901 }
3902
3903 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
3904 if (err)
3905 return err;
3906 }
3907
3908 return 0;
3909}
3910
3911static int mask_virtual_engine(struct intel_gt *gt,
3912 struct intel_engine_cs **siblings,
3913 unsigned int nsibling)
3914{
3915 struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
3916 struct intel_context *ve;
3917 struct igt_live_test t;
3918 unsigned int n;
3919 int err;
3920
3921
3922
3923
3924
3925
3926 ve = intel_execlists_create_virtual(siblings, nsibling);
3927 if (IS_ERR(ve)) {
3928 err = PTR_ERR(ve);
3929 goto out_close;
3930 }
3931
3932 err = intel_context_pin(ve);
3933 if (err)
3934 goto out_put;
3935
3936 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
3937 if (err)
3938 goto out_unpin;
3939
3940 for (n = 0; n < nsibling; n++) {
3941 request[n] = i915_request_create(ve);
3942 if (IS_ERR(request[n])) {
3943 err = PTR_ERR(request[n]);
3944 nsibling = n;
3945 goto out;
3946 }
3947
3948
3949 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3950
3951 i915_request_get(request[n]);
3952 i915_request_add(request[n]);
3953 }
3954
3955 for (n = 0; n < nsibling; n++) {
3956 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
3957 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3958 __func__, ve->engine->name,
3959 request[n]->fence.context,
3960 request[n]->fence.seqno);
3961
3962 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3963 __func__, ve->engine->name,
3964 request[n]->fence.context,
3965 request[n]->fence.seqno);
3966 GEM_TRACE_DUMP();
3967 intel_gt_set_wedged(gt);
3968 err = -EIO;
3969 goto out;
3970 }
3971
3972 if (request[n]->engine != siblings[nsibling - n - 1]) {
3973 pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3974 request[n]->engine->name,
3975 siblings[nsibling - n - 1]->name);
3976 err = -EINVAL;
3977 goto out;
3978 }
3979 }
3980
3981 err = igt_live_test_end(&t);
3982out:
3983 if (igt_flush_test(gt->i915))
3984 err = -EIO;
3985
3986 for (n = 0; n < nsibling; n++)
3987 i915_request_put(request[n]);
3988
3989out_unpin:
3990 intel_context_unpin(ve);
3991out_put:
3992 intel_context_put(ve);
3993out_close:
3994 return err;
3995}
3996
3997static int live_virtual_mask(void *arg)
3998{
3999 struct intel_gt *gt = arg;
4000 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4001 unsigned int class;
4002 int err;
4003
4004 if (intel_uc_uses_guc_submission(>->uc))
4005 return 0;
4006
4007 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4008 unsigned int nsibling;
4009
4010 nsibling = select_siblings(gt, class, siblings);
4011 if (nsibling < 2)
4012 continue;
4013
4014 err = mask_virtual_engine(gt, siblings, nsibling);
4015 if (err)
4016 return err;
4017 }
4018
4019 return 0;
4020}
4021
4022static int slicein_virtual_engine(struct intel_gt *gt,
4023 struct intel_engine_cs **siblings,
4024 unsigned int nsibling)
4025{
4026 const long timeout = slice_timeout(siblings[0]);
4027 struct intel_context *ce;
4028 struct i915_request *rq;
4029 struct igt_spinner spin;
4030 unsigned int n;
4031 int err = 0;
4032
4033
4034
4035
4036
4037 if (igt_spinner_init(&spin, gt))
4038 return -ENOMEM;
4039
4040 for (n = 0; n < nsibling; n++) {
4041 ce = intel_context_create(siblings[n]);
4042 if (IS_ERR(ce)) {
4043 err = PTR_ERR(ce);
4044 goto out;
4045 }
4046
4047 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4048 intel_context_put(ce);
4049 if (IS_ERR(rq)) {
4050 err = PTR_ERR(rq);
4051 goto out;
4052 }
4053
4054 i915_request_add(rq);
4055 }
4056
4057 ce = intel_execlists_create_virtual(siblings, nsibling);
4058 if (IS_ERR(ce)) {
4059 err = PTR_ERR(ce);
4060 goto out;
4061 }
4062
4063 rq = intel_context_create_request(ce);
4064 intel_context_put(ce);
4065 if (IS_ERR(rq)) {
4066 err = PTR_ERR(rq);
4067 goto out;
4068 }
4069
4070 i915_request_get(rq);
4071 i915_request_add(rq);
4072 if (i915_request_wait(rq, 0, timeout) < 0) {
4073 GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n",
4074 __func__, rq->engine->name);
4075 GEM_TRACE_DUMP();
4076 intel_gt_set_wedged(gt);
4077 err = -EIO;
4078 }
4079 i915_request_put(rq);
4080
4081out:
4082 igt_spinner_end(&spin);
4083 if (igt_flush_test(gt->i915))
4084 err = -EIO;
4085 igt_spinner_fini(&spin);
4086 return err;
4087}
4088
4089static int sliceout_virtual_engine(struct intel_gt *gt,
4090 struct intel_engine_cs **siblings,
4091 unsigned int nsibling)
4092{
4093 const long timeout = slice_timeout(siblings[0]);
4094 struct intel_context *ce;
4095 struct i915_request *rq;
4096 struct igt_spinner spin;
4097 unsigned int n;
4098 int err = 0;
4099
4100
4101
4102
4103
4104 if (igt_spinner_init(&spin, gt))
4105 return -ENOMEM;
4106
4107
4108 for (n = 0; n < nsibling; n++) {
4109 ce = intel_execlists_create_virtual(siblings, nsibling);
4110 if (IS_ERR(ce)) {
4111 err = PTR_ERR(ce);
4112 goto out;
4113 }
4114
4115 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
4116 intel_context_put(ce);
4117 if (IS_ERR(rq)) {
4118 err = PTR_ERR(rq);
4119 goto out;
4120 }
4121
4122 i915_request_add(rq);
4123 }
4124
4125 for (n = 0; !err && n < nsibling; n++) {
4126 ce = intel_context_create(siblings[n]);
4127 if (IS_ERR(ce)) {
4128 err = PTR_ERR(ce);
4129 goto out;
4130 }
4131
4132 rq = intel_context_create_request(ce);
4133 intel_context_put(ce);
4134 if (IS_ERR(rq)) {
4135 err = PTR_ERR(rq);
4136 goto out;
4137 }
4138
4139 i915_request_get(rq);
4140 i915_request_add(rq);
4141 if (i915_request_wait(rq, 0, timeout) < 0) {
4142 GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n",
4143 __func__, siblings[n]->name);
4144 GEM_TRACE_DUMP();
4145 intel_gt_set_wedged(gt);
4146 err = -EIO;
4147 }
4148 i915_request_put(rq);
4149 }
4150
4151out:
4152 igt_spinner_end(&spin);
4153 if (igt_flush_test(gt->i915))
4154 err = -EIO;
4155 igt_spinner_fini(&spin);
4156 return err;
4157}
4158
4159static int live_virtual_slice(void *arg)
4160{
4161 struct intel_gt *gt = arg;
4162 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4163 unsigned int class;
4164 int err;
4165
4166 if (intel_uc_uses_guc_submission(>->uc))
4167 return 0;
4168
4169 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4170 unsigned int nsibling;
4171
4172 nsibling = __select_siblings(gt, class, siblings,
4173 intel_engine_has_timeslices);
4174 if (nsibling < 2)
4175 continue;
4176
4177 err = slicein_virtual_engine(gt, siblings, nsibling);
4178 if (err)
4179 return err;
4180
4181 err = sliceout_virtual_engine(gt, siblings, nsibling);
4182 if (err)
4183 return err;
4184 }
4185
4186 return 0;
4187}
4188
4189static int preserved_virtual_engine(struct intel_gt *gt,
4190 struct intel_engine_cs **siblings,
4191 unsigned int nsibling)
4192{
4193 struct i915_request *last = NULL;
4194 struct intel_context *ve;
4195 struct i915_vma *scratch;
4196 struct igt_live_test t;
4197 unsigned int n;
4198 int err = 0;
4199 u32 *cs;
4200
4201 scratch =
4202 __vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,
4203 PAGE_SIZE);
4204 if (IS_ERR(scratch))
4205 return PTR_ERR(scratch);
4206
4207 err = i915_vma_sync(scratch);
4208 if (err)
4209 goto out_scratch;
4210
4211 ve = intel_execlists_create_virtual(siblings, nsibling);
4212 if (IS_ERR(ve)) {
4213 err = PTR_ERR(ve);
4214 goto out_scratch;
4215 }
4216
4217 err = intel_context_pin(ve);
4218 if (err)
4219 goto out_put;
4220
4221 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
4222 if (err)
4223 goto out_unpin;
4224
4225 for (n = 0; n < NUM_GPR_DW; n++) {
4226 struct intel_engine_cs *engine = siblings[n % nsibling];
4227 struct i915_request *rq;
4228
4229 rq = i915_request_create(ve);
4230 if (IS_ERR(rq)) {
4231 err = PTR_ERR(rq);
4232 goto out_end;
4233 }
4234
4235 i915_request_put(last);
4236 last = i915_request_get(rq);
4237
4238 cs = intel_ring_begin(rq, 8);
4239 if (IS_ERR(cs)) {
4240 i915_request_add(rq);
4241 err = PTR_ERR(cs);
4242 goto out_end;
4243 }
4244
4245 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4246 *cs++ = CS_GPR(engine, n);
4247 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4248 *cs++ = 0;
4249
4250 *cs++ = MI_LOAD_REGISTER_IMM(1);
4251 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
4252 *cs++ = n + 1;
4253
4254 *cs++ = MI_NOOP;
4255 intel_ring_advance(rq, cs);
4256
4257
4258 rq->execution_mask = engine->mask;
4259 i915_request_add(rq);
4260 }
4261
4262 if (i915_request_wait(last, 0, HZ / 5) < 0) {
4263 err = -ETIME;
4264 goto out_end;
4265 }
4266
4267 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
4268 if (IS_ERR(cs)) {
4269 err = PTR_ERR(cs);
4270 goto out_end;
4271 }
4272
4273 for (n = 0; n < NUM_GPR_DW; n++) {
4274 if (cs[n] != n) {
4275 pr_err("Incorrect value[%d] found for GPR[%d]\n",
4276 cs[n], n);
4277 err = -EINVAL;
4278 break;
4279 }
4280 }
4281
4282 i915_gem_object_unpin_map(scratch->obj);
4283
4284out_end:
4285 if (igt_live_test_end(&t))
4286 err = -EIO;
4287 i915_request_put(last);
4288out_unpin:
4289 intel_context_unpin(ve);
4290out_put:
4291 intel_context_put(ve);
4292out_scratch:
4293 i915_vma_unpin_and_release(&scratch, 0);
4294 return err;
4295}
4296
4297static int live_virtual_preserved(void *arg)
4298{
4299 struct intel_gt *gt = arg;
4300 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4301 unsigned int class;
4302
4303
4304
4305
4306
4307
4308
4309 if (intel_uc_uses_guc_submission(>->uc))
4310 return 0;
4311
4312
4313 if (GRAPHICS_VER(gt->i915) < 9)
4314 return 0;
4315
4316 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4317 int nsibling, err;
4318
4319 nsibling = select_siblings(gt, class, siblings);
4320 if (nsibling < 2)
4321 continue;
4322
4323 err = preserved_virtual_engine(gt, siblings, nsibling);
4324 if (err)
4325 return err;
4326 }
4327
4328 return 0;
4329}
4330
4331static int bond_virtual_engine(struct intel_gt *gt,
4332 unsigned int class,
4333 struct intel_engine_cs **siblings,
4334 unsigned int nsibling,
4335 unsigned int flags)
4336#define BOND_SCHEDULE BIT(0)
4337{
4338 struct intel_engine_cs *master;
4339 struct i915_request *rq[16];
4340 enum intel_engine_id id;
4341 struct igt_spinner spin;
4342 unsigned long n;
4343 int err;
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384 GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1);
4385
4386 if (igt_spinner_init(&spin, gt))
4387 return -ENOMEM;
4388
4389 err = 0;
4390 rq[0] = ERR_PTR(-ENOMEM);
4391 for_each_engine(master, gt, id) {
4392 struct i915_sw_fence fence = {};
4393 struct intel_context *ce;
4394
4395 if (master->class == class)
4396 continue;
4397
4398 ce = intel_context_create(master);
4399 if (IS_ERR(ce)) {
4400 err = PTR_ERR(ce);
4401 goto out;
4402 }
4403
4404 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq));
4405
4406 rq[0] = igt_spinner_create_request(&spin, ce, MI_NOOP);
4407 intel_context_put(ce);
4408 if (IS_ERR(rq[0])) {
4409 err = PTR_ERR(rq[0]);
4410 goto out;
4411 }
4412 i915_request_get(rq[0]);
4413
4414 if (flags & BOND_SCHEDULE) {
4415 onstack_fence_init(&fence);
4416 err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit,
4417 &fence,
4418 GFP_KERNEL);
4419 }
4420
4421 i915_request_add(rq[0]);
4422 if (err < 0)
4423 goto out;
4424
4425 if (!(flags & BOND_SCHEDULE) &&
4426 !igt_wait_for_spinner(&spin, rq[0])) {
4427 err = -EIO;
4428 goto out;
4429 }
4430
4431 for (n = 0; n < nsibling; n++) {
4432 struct intel_context *ve;
4433
4434 ve = intel_execlists_create_virtual(siblings, nsibling);
4435 if (IS_ERR(ve)) {
4436 err = PTR_ERR(ve);
4437 onstack_fence_fini(&fence);
4438 goto out;
4439 }
4440
4441 err = intel_virtual_engine_attach_bond(ve->engine,
4442 master,
4443 siblings[n]);
4444 if (err) {
4445 intel_context_put(ve);
4446 onstack_fence_fini(&fence);
4447 goto out;
4448 }
4449
4450 err = intel_context_pin(ve);
4451 intel_context_put(ve);
4452 if (err) {
4453 onstack_fence_fini(&fence);
4454 goto out;
4455 }
4456
4457 rq[n + 1] = i915_request_create(ve);
4458 intel_context_unpin(ve);
4459 if (IS_ERR(rq[n + 1])) {
4460 err = PTR_ERR(rq[n + 1]);
4461 onstack_fence_fini(&fence);
4462 goto out;
4463 }
4464 i915_request_get(rq[n + 1]);
4465
4466 err = i915_request_await_execution(rq[n + 1],
4467 &rq[0]->fence,
4468 ve->engine->bond_execute);
4469 i915_request_add(rq[n + 1]);
4470 if (err < 0) {
4471 onstack_fence_fini(&fence);
4472 goto out;
4473 }
4474 }
4475 onstack_fence_fini(&fence);
4476 intel_engine_flush_submission(master);
4477 igt_spinner_end(&spin);
4478
4479 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) {
4480 pr_err("Master request did not execute (on %s)!\n",
4481 rq[0]->engine->name);
4482 err = -EIO;
4483 goto out;
4484 }
4485
4486 for (n = 0; n < nsibling; n++) {
4487 if (i915_request_wait(rq[n + 1], 0,
4488 MAX_SCHEDULE_TIMEOUT) < 0) {
4489 err = -EIO;
4490 goto out;
4491 }
4492
4493 if (rq[n + 1]->engine != siblings[n]) {
4494 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n",
4495 siblings[n]->name,
4496 rq[n + 1]->engine->name,
4497 rq[0]->engine->name);
4498 err = -EINVAL;
4499 goto out;
4500 }
4501 }
4502
4503 for (n = 0; !IS_ERR(rq[n]); n++)
4504 i915_request_put(rq[n]);
4505 rq[0] = ERR_PTR(-ENOMEM);
4506 }
4507
4508out:
4509 for (n = 0; !IS_ERR(rq[n]); n++)
4510 i915_request_put(rq[n]);
4511 if (igt_flush_test(gt->i915))
4512 err = -EIO;
4513
4514 igt_spinner_fini(&spin);
4515 return err;
4516}
4517
4518static int live_virtual_bond(void *arg)
4519{
4520 static const struct phase {
4521 const char *name;
4522 unsigned int flags;
4523 } phases[] = {
4524 { "", 0 },
4525 { "schedule", BOND_SCHEDULE },
4526 { },
4527 };
4528 struct intel_gt *gt = arg;
4529 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4530 unsigned int class;
4531 int err;
4532
4533 if (intel_uc_uses_guc_submission(>->uc))
4534 return 0;
4535
4536 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4537 const struct phase *p;
4538 int nsibling;
4539
4540 nsibling = select_siblings(gt, class, siblings);
4541 if (nsibling < 2)
4542 continue;
4543
4544 for (p = phases; p->name; p++) {
4545 err = bond_virtual_engine(gt,
4546 class, siblings, nsibling,
4547 p->flags);
4548 if (err) {
4549 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n",
4550 __func__, p->name, class, nsibling, err);
4551 return err;
4552 }
4553 }
4554 }
4555
4556 return 0;
4557}
4558
4559static int reset_virtual_engine(struct intel_gt *gt,
4560 struct intel_engine_cs **siblings,
4561 unsigned int nsibling)
4562{
4563 struct intel_engine_cs *engine;
4564 struct intel_context *ve;
4565 struct igt_spinner spin;
4566 struct i915_request *rq;
4567 unsigned int n;
4568 int err = 0;
4569
4570
4571
4572
4573
4574
4575
4576 if (igt_spinner_init(&spin, gt))
4577 return -ENOMEM;
4578
4579 ve = intel_execlists_create_virtual(siblings, nsibling);
4580 if (IS_ERR(ve)) {
4581 err = PTR_ERR(ve);
4582 goto out_spin;
4583 }
4584
4585 for (n = 0; n < nsibling; n++)
4586 st_engine_heartbeat_disable(siblings[n]);
4587
4588 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
4589 if (IS_ERR(rq)) {
4590 err = PTR_ERR(rq);
4591 goto out_heartbeat;
4592 }
4593 i915_request_add(rq);
4594
4595 if (!igt_wait_for_spinner(&spin, rq)) {
4596 intel_gt_set_wedged(gt);
4597 err = -ETIME;
4598 goto out_heartbeat;
4599 }
4600
4601 engine = rq->engine;
4602 GEM_BUG_ON(engine == ve->engine);
4603
4604
4605 err = engine_lock_reset_tasklet(engine);
4606 if (err)
4607 goto out_heartbeat;
4608
4609 engine->execlists.tasklet.callback(&engine->execlists.tasklet);
4610 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
4611
4612
4613 spin_lock_irq(&engine->active.lock);
4614 __unwind_incomplete_requests(engine);
4615 spin_unlock_irq(&engine->active.lock);
4616 GEM_BUG_ON(rq->engine != engine);
4617
4618
4619 execlists_hold(engine, rq);
4620 GEM_BUG_ON(!i915_request_on_hold(rq));
4621
4622 __intel_engine_reset_bh(engine, NULL);
4623 GEM_BUG_ON(rq->fence.error != -EIO);
4624
4625
4626 engine_unlock_reset_tasklet(engine);
4627
4628
4629 i915_request_get(rq);
4630 if (!i915_request_wait(rq, 0, HZ / 5)) {
4631 pr_err("%s: on hold request completed!\n",
4632 engine->name);
4633 intel_gt_set_wedged(gt);
4634 err = -EIO;
4635 goto out_rq;
4636 }
4637 GEM_BUG_ON(!i915_request_on_hold(rq));
4638
4639
4640 execlists_unhold(engine, rq);
4641 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4642 pr_err("%s: held request did not complete!\n",
4643 engine->name);
4644 intel_gt_set_wedged(gt);
4645 err = -ETIME;
4646 }
4647
4648out_rq:
4649 i915_request_put(rq);
4650out_heartbeat:
4651 for (n = 0; n < nsibling; n++)
4652 st_engine_heartbeat_enable(siblings[n]);
4653
4654 intel_context_put(ve);
4655out_spin:
4656 igt_spinner_fini(&spin);
4657 return err;
4658}
4659
4660static int live_virtual_reset(void *arg)
4661{
4662 struct intel_gt *gt = arg;
4663 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
4664 unsigned int class;
4665
4666
4667
4668
4669
4670
4671
4672
4673 if (intel_uc_uses_guc_submission(>->uc))
4674 return 0;
4675
4676 if (!intel_has_reset_engine(gt))
4677 return 0;
4678
4679 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
4680 int nsibling, err;
4681
4682 nsibling = select_siblings(gt, class, siblings);
4683 if (nsibling < 2)
4684 continue;
4685
4686 err = reset_virtual_engine(gt, siblings, nsibling);
4687 if (err)
4688 return err;
4689 }
4690
4691 return 0;
4692}
4693
4694int intel_execlists_live_selftests(struct drm_i915_private *i915)
4695{
4696 static const struct i915_subtest tests[] = {
4697 SUBTEST(live_sanitycheck),
4698 SUBTEST(live_unlite_switch),
4699 SUBTEST(live_unlite_preempt),
4700 SUBTEST(live_unlite_ring),
4701 SUBTEST(live_pin_rewind),
4702 SUBTEST(live_hold_reset),
4703 SUBTEST(live_error_interrupt),
4704 SUBTEST(live_timeslice_preempt),
4705 SUBTEST(live_timeslice_rewind),
4706 SUBTEST(live_timeslice_queue),
4707 SUBTEST(live_timeslice_nopreempt),
4708 SUBTEST(live_busywait_preempt),
4709 SUBTEST(live_preempt),
4710 SUBTEST(live_late_preempt),
4711 SUBTEST(live_nopreempt),
4712 SUBTEST(live_preempt_cancel),
4713 SUBTEST(live_suppress_self_preempt),
4714 SUBTEST(live_chain_preempt),
4715 SUBTEST(live_preempt_ring),
4716 SUBTEST(live_preempt_gang),
4717 SUBTEST(live_preempt_timeout),
4718 SUBTEST(live_preempt_user),
4719 SUBTEST(live_preempt_smoke),
4720 SUBTEST(live_virtual_engine),
4721 SUBTEST(live_virtual_mask),
4722 SUBTEST(live_virtual_preserved),
4723 SUBTEST(live_virtual_slice),
4724 SUBTEST(live_virtual_bond),
4725 SUBTEST(live_virtual_reset),
4726 };
4727
4728 if (i915->gt.submission_method != INTEL_SUBMISSION_ELSP)
4729 return 0;
4730
4731 if (intel_gt_is_wedged(&i915->gt))
4732 return 0;
4733
4734 return intel_gt_live_subtests(tests, &i915->gt);
4735}
4736