1
2
3
4
5
6#include "gem/i915_gem_internal.h"
7#include "gem/i915_gem_pm.h"
8#include "gt/intel_engine_user.h"
9#include "gt/intel_gt.h"
10#include "i915_selftest.h"
11#include "intel_reset.h"
12
13#include "selftests/igt_flush_test.h"
14#include "selftests/igt_reset.h"
15#include "selftests/igt_spinner.h"
16#include "selftests/intel_scheduler_helpers.h"
17#include "selftests/mock_drm.h"
18
19#include "gem/selftests/igt_gem_utils.h"
20#include "gem/selftests/mock_context.h"
21
22static const struct wo_register {
23 enum intel_platform platform;
24 u32 reg;
25} wo_registers[] = {
26 { INTEL_GEMINILAKE, 0x731c }
27};
28
29struct wa_lists {
30 struct i915_wa_list gt_wa_list;
31 struct {
32 struct i915_wa_list wa_list;
33 struct i915_wa_list ctx_wa_list;
34 } engine[I915_NUM_ENGINES];
35};
36
37static int request_add_sync(struct i915_request *rq, int err)
38{
39 i915_request_get(rq);
40 i915_request_add(rq);
41 if (i915_request_wait(rq, 0, HZ / 5) < 0)
42 err = -EIO;
43 i915_request_put(rq);
44
45 return err;
46}
47
48static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
49{
50 int err = 0;
51
52 i915_request_get(rq);
53 i915_request_add(rq);
54 if (spin && !igt_wait_for_spinner(spin, rq))
55 err = -ETIMEDOUT;
56 i915_request_put(rq);
57
58 return err;
59}
60
61static void
62reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
63{
64 struct intel_engine_cs *engine;
65 enum intel_engine_id id;
66
67 memset(lists, 0, sizeof(*lists));
68
69 wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
70 gt_init_workarounds(gt, &lists->gt_wa_list);
71 wa_init_finish(&lists->gt_wa_list);
72
73 for_each_engine(engine, gt, id) {
74 struct i915_wa_list *wal = &lists->engine[id].wa_list;
75
76 wa_init_start(wal, "REF", engine->name);
77 engine_init_workarounds(engine, wal);
78 wa_init_finish(wal);
79
80 __intel_engine_init_ctx_wa(engine,
81 &lists->engine[id].ctx_wa_list,
82 "CTX_REF");
83 }
84}
85
86static void
87reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
88{
89 struct intel_engine_cs *engine;
90 enum intel_engine_id id;
91
92 for_each_engine(engine, gt, id)
93 intel_wa_list_free(&lists->engine[id].wa_list);
94
95 intel_wa_list_free(&lists->gt_wa_list);
96}
97
98static struct drm_i915_gem_object *
99read_nonprivs(struct intel_context *ce)
100{
101 struct intel_engine_cs *engine = ce->engine;
102 const u32 base = engine->mmio_base;
103 struct drm_i915_gem_object *result;
104 struct i915_request *rq;
105 struct i915_vma *vma;
106 u32 srm, *cs;
107 int err;
108 int i;
109
110 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
111 if (IS_ERR(result))
112 return result;
113
114 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
115
116 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
117 if (IS_ERR(cs)) {
118 err = PTR_ERR(cs);
119 goto err_obj;
120 }
121 memset(cs, 0xc5, PAGE_SIZE);
122 i915_gem_object_flush_map(result);
123 i915_gem_object_unpin_map(result);
124
125 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
126 if (IS_ERR(vma)) {
127 err = PTR_ERR(vma);
128 goto err_obj;
129 }
130
131 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
132 if (err)
133 goto err_obj;
134
135 rq = intel_context_create_request(ce);
136 if (IS_ERR(rq)) {
137 err = PTR_ERR(rq);
138 goto err_pin;
139 }
140
141 i915_vma_lock(vma);
142 err = i915_request_await_object(rq, vma->obj, true);
143 if (err == 0)
144 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
145 i915_vma_unlock(vma);
146 if (err)
147 goto err_req;
148
149 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
150 if (GRAPHICS_VER(engine->i915) >= 8)
151 srm++;
152
153 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
154 if (IS_ERR(cs)) {
155 err = PTR_ERR(cs);
156 goto err_req;
157 }
158
159 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
160 *cs++ = srm;
161 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
162 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
163 *cs++ = 0;
164 }
165 intel_ring_advance(rq, cs);
166
167 i915_request_add(rq);
168 i915_vma_unpin(vma);
169
170 return result;
171
172err_req:
173 i915_request_add(rq);
174err_pin:
175 i915_vma_unpin(vma);
176err_obj:
177 i915_gem_object_put(result);
178 return ERR_PTR(err);
179}
180
181static u32
182get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
183{
184 i915_reg_t reg = i < engine->whitelist.count ?
185 engine->whitelist.list[i].reg :
186 RING_NOPID(engine->mmio_base);
187
188 return i915_mmio_reg_offset(reg);
189}
190
191static void
192print_results(const struct intel_engine_cs *engine, const u32 *results)
193{
194 unsigned int i;
195
196 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
197 u32 expected = get_whitelist_reg(engine, i);
198 u32 actual = results[i];
199
200 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
201 i, expected, actual);
202 }
203}
204
205static int check_whitelist(struct intel_context *ce)
206{
207 struct intel_engine_cs *engine = ce->engine;
208 struct drm_i915_gem_object *results;
209 struct intel_wedge_me wedge;
210 u32 *vaddr;
211 int err;
212 int i;
213
214 results = read_nonprivs(ce);
215 if (IS_ERR(results))
216 return PTR_ERR(results);
217
218 err = 0;
219 i915_gem_object_lock(results, NULL);
220 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5)
221 err = i915_gem_object_set_to_cpu_domain(results, false);
222
223 if (intel_gt_is_wedged(engine->gt))
224 err = -EIO;
225 if (err)
226 goto out_put;
227
228 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
229 if (IS_ERR(vaddr)) {
230 err = PTR_ERR(vaddr);
231 goto out_put;
232 }
233
234 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
235 u32 expected = get_whitelist_reg(engine, i);
236 u32 actual = vaddr[i];
237
238 if (expected != actual) {
239 print_results(engine, vaddr);
240 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
241 i, expected, actual);
242
243 err = -EINVAL;
244 break;
245 }
246 }
247
248 i915_gem_object_unpin_map(results);
249out_put:
250 i915_gem_object_unlock(results);
251 i915_gem_object_put(results);
252 return err;
253}
254
255static int do_device_reset(struct intel_engine_cs *engine)
256{
257 intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
258 return 0;
259}
260
261static int do_engine_reset(struct intel_engine_cs *engine)
262{
263 return intel_engine_reset(engine, "live_workarounds");
264}
265
266static int do_guc_reset(struct intel_engine_cs *engine)
267{
268
269 return 0;
270}
271
272static int
273switch_to_scratch_context(struct intel_engine_cs *engine,
274 struct igt_spinner *spin,
275 struct i915_request **rq)
276{
277 struct intel_context *ce;
278 int err = 0;
279
280 ce = intel_context_create(engine);
281 if (IS_ERR(ce))
282 return PTR_ERR(ce);
283
284 *rq = igt_spinner_create_request(spin, ce, MI_NOOP);
285 intel_context_put(ce);
286
287 if (IS_ERR(*rq)) {
288 spin = NULL;
289 err = PTR_ERR(*rq);
290 goto err;
291 }
292
293 err = request_add_spin(*rq, spin);
294err:
295 if (err && spin)
296 igt_spinner_end(spin);
297
298 return err;
299}
300
301static int check_whitelist_across_reset(struct intel_engine_cs *engine,
302 int (*reset)(struct intel_engine_cs *),
303 const char *name)
304{
305 struct intel_context *ce, *tmp;
306 struct igt_spinner spin;
307 struct i915_request *rq;
308 intel_wakeref_t wakeref;
309 int err;
310
311 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
312 engine->whitelist.count, engine->name, name);
313
314 ce = intel_context_create(engine);
315 if (IS_ERR(ce))
316 return PTR_ERR(ce);
317
318 err = igt_spinner_init(&spin, engine->gt);
319 if (err)
320 goto out_ctx;
321
322 err = check_whitelist(ce);
323 if (err) {
324 pr_err("Invalid whitelist *before* %s reset!\n", name);
325 goto out_spin;
326 }
327
328 err = switch_to_scratch_context(engine, &spin, &rq);
329 if (err)
330 goto out_spin;
331
332
333 if (i915_request_completed(rq)) {
334 pr_err("%s spinner failed to start\n", name);
335 err = -ETIMEDOUT;
336 goto out_spin;
337 }
338
339 with_intel_runtime_pm(engine->uncore->rpm, wakeref)
340 err = reset(engine);
341
342
343 if (err == 0)
344 err = intel_selftest_wait_for_rq(rq);
345
346 igt_spinner_end(&spin);
347
348 if (err) {
349 pr_err("%s reset failed\n", name);
350 goto out_spin;
351 }
352
353 err = check_whitelist(ce);
354 if (err) {
355 pr_err("Whitelist not preserved in context across %s reset!\n",
356 name);
357 goto out_spin;
358 }
359
360 tmp = intel_context_create(engine);
361 if (IS_ERR(tmp)) {
362 err = PTR_ERR(tmp);
363 goto out_spin;
364 }
365 intel_context_put(ce);
366 ce = tmp;
367
368 err = check_whitelist(ce);
369 if (err) {
370 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
371 name);
372 goto out_spin;
373 }
374
375out_spin:
376 igt_spinner_fini(&spin);
377out_ctx:
378 intel_context_put(ce);
379 return err;
380}
381
382static struct i915_vma *create_batch(struct i915_address_space *vm)
383{
384 struct drm_i915_gem_object *obj;
385 struct i915_vma *vma;
386 int err;
387
388 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
389 if (IS_ERR(obj))
390 return ERR_CAST(obj);
391
392 vma = i915_vma_instance(obj, vm, NULL);
393 if (IS_ERR(vma)) {
394 err = PTR_ERR(vma);
395 goto err_obj;
396 }
397
398 err = i915_vma_pin(vma, 0, 0, PIN_USER);
399 if (err)
400 goto err_obj;
401
402 return vma;
403
404err_obj:
405 i915_gem_object_put(obj);
406 return ERR_PTR(err);
407}
408
409static u32 reg_write(u32 old, u32 new, u32 rsvd)
410{
411 if (rsvd == 0x0000ffff) {
412 old &= ~(new >> 16);
413 old |= new & (new >> 16);
414 } else {
415 old &= ~rsvd;
416 old |= new & rsvd;
417 }
418
419 return old;
420}
421
422static bool wo_register(struct intel_engine_cs *engine, u32 reg)
423{
424 enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
425 int i;
426
427 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
428 RING_FORCE_TO_NONPRIV_ACCESS_WR)
429 return true;
430
431 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
432 if (wo_registers[i].platform == platform &&
433 wo_registers[i].reg == reg)
434 return true;
435 }
436
437 return false;
438}
439
440static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
441{
442 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
443 switch (reg) {
444 case 0x358:
445 case 0x35c:
446 case 0x3a8:
447 return true;
448
449 default:
450 return false;
451 }
452}
453
454static bool ro_register(u32 reg)
455{
456 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
457 RING_FORCE_TO_NONPRIV_ACCESS_RD)
458 return true;
459
460 return false;
461}
462
463static int whitelist_writable_count(struct intel_engine_cs *engine)
464{
465 int count = engine->whitelist.count;
466 int i;
467
468 for (i = 0; i < engine->whitelist.count; i++) {
469 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
470
471 if (ro_register(reg))
472 count--;
473 }
474
475 return count;
476}
477
478static int check_dirty_whitelist(struct intel_context *ce)
479{
480 const u32 values[] = {
481 0x00000000,
482 0x01010101,
483 0x10100101,
484 0x03030303,
485 0x30300303,
486 0x05050505,
487 0x50500505,
488 0x0f0f0f0f,
489 0xf00ff00f,
490 0x10101010,
491 0xf0f01010,
492 0x30303030,
493 0xa0a03030,
494 0x50505050,
495 0xc0c05050,
496 0xf0f0f0f0,
497 0x11111111,
498 0x33333333,
499 0x55555555,
500 0x0000ffff,
501 0x00ff00ff,
502 0xff0000ff,
503 0xffff00ff,
504 0xffffffff,
505 };
506 struct intel_engine_cs *engine = ce->engine;
507 struct i915_vma *scratch;
508 struct i915_vma *batch;
509 int err = 0, i, v, sz;
510 u32 *cs, *results;
511
512 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
513 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
514 if (IS_ERR(scratch))
515 return PTR_ERR(scratch);
516
517 batch = create_batch(ce->vm);
518 if (IS_ERR(batch)) {
519 err = PTR_ERR(batch);
520 goto out_scratch;
521 }
522
523 for (i = 0; i < engine->whitelist.count; i++) {
524 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
525 struct i915_gem_ww_ctx ww;
526 u64 addr = scratch->node.start;
527 struct i915_request *rq;
528 u32 srm, lrm, rsvd;
529 u32 expect;
530 int idx;
531 bool ro_reg;
532
533 if (wo_register(engine, reg))
534 continue;
535
536 if (timestamp(engine, reg))
537 continue;
538
539 ro_reg = ro_register(reg);
540
541 i915_gem_ww_ctx_init(&ww, false);
542retry:
543 cs = NULL;
544 err = i915_gem_object_lock(scratch->obj, &ww);
545 if (!err)
546 err = i915_gem_object_lock(batch->obj, &ww);
547 if (!err)
548 err = intel_context_pin_ww(ce, &ww);
549 if (err)
550 goto out;
551
552 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
553 if (IS_ERR(cs)) {
554 err = PTR_ERR(cs);
555 goto out_ctx;
556 }
557
558 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
559 if (IS_ERR(results)) {
560 err = PTR_ERR(results);
561 goto out_unmap_batch;
562 }
563
564
565 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
566
567 srm = MI_STORE_REGISTER_MEM;
568 lrm = MI_LOAD_REGISTER_MEM;
569 if (GRAPHICS_VER(engine->i915) >= 8)
570 lrm++, srm++;
571
572 pr_debug("%s: Writing garbage to %x\n",
573 engine->name, reg);
574
575
576 *cs++ = srm;
577 *cs++ = reg;
578 *cs++ = lower_32_bits(addr);
579 *cs++ = upper_32_bits(addr);
580
581 idx = 1;
582 for (v = 0; v < ARRAY_SIZE(values); v++) {
583
584 *cs++ = MI_LOAD_REGISTER_IMM(1);
585 *cs++ = reg;
586 *cs++ = values[v];
587
588
589 *cs++ = srm;
590 *cs++ = reg;
591 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
592 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
593 idx++;
594 }
595 for (v = 0; v < ARRAY_SIZE(values); v++) {
596
597 *cs++ = MI_LOAD_REGISTER_IMM(1);
598 *cs++ = reg;
599 *cs++ = ~values[v];
600
601
602 *cs++ = srm;
603 *cs++ = reg;
604 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
605 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
606 idx++;
607 }
608 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
609
610
611 *cs++ = lrm;
612 *cs++ = reg;
613 *cs++ = lower_32_bits(addr);
614 *cs++ = upper_32_bits(addr);
615
616 *cs++ = MI_BATCH_BUFFER_END;
617
618 i915_gem_object_flush_map(batch->obj);
619 i915_gem_object_unpin_map(batch->obj);
620 intel_gt_chipset_flush(engine->gt);
621 cs = NULL;
622
623 rq = i915_request_create(ce);
624 if (IS_ERR(rq)) {
625 err = PTR_ERR(rq);
626 goto out_unmap_scratch;
627 }
628
629 if (engine->emit_init_breadcrumb) {
630 err = engine->emit_init_breadcrumb(rq);
631 if (err)
632 goto err_request;
633 }
634
635 err = i915_request_await_object(rq, batch->obj, false);
636 if (err == 0)
637 err = i915_vma_move_to_active(batch, rq, 0);
638 if (err)
639 goto err_request;
640
641 err = i915_request_await_object(rq, scratch->obj, true);
642 if (err == 0)
643 err = i915_vma_move_to_active(scratch, rq,
644 EXEC_OBJECT_WRITE);
645 if (err)
646 goto err_request;
647
648 err = engine->emit_bb_start(rq,
649 batch->node.start, PAGE_SIZE,
650 0);
651 if (err)
652 goto err_request;
653
654err_request:
655 err = request_add_sync(rq, err);
656 if (err) {
657 pr_err("%s: Futzing %x timedout; cancelling test\n",
658 engine->name, reg);
659 intel_gt_set_wedged(engine->gt);
660 goto out_unmap_scratch;
661 }
662
663 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
664 if (!ro_reg) {
665
666 rsvd = results[ARRAY_SIZE(values)];
667 if (!rsvd) {
668 pr_err("%s: Unable to write to whitelisted register %x\n",
669 engine->name, reg);
670 err = -EINVAL;
671 goto out_unmap_scratch;
672 }
673 } else {
674 rsvd = 0;
675 }
676
677 expect = results[0];
678 idx = 1;
679 for (v = 0; v < ARRAY_SIZE(values); v++) {
680 if (ro_reg)
681 expect = results[0];
682 else
683 expect = reg_write(expect, values[v], rsvd);
684
685 if (results[idx] != expect)
686 err++;
687 idx++;
688 }
689 for (v = 0; v < ARRAY_SIZE(values); v++) {
690 if (ro_reg)
691 expect = results[0];
692 else
693 expect = reg_write(expect, ~values[v], rsvd);
694
695 if (results[idx] != expect)
696 err++;
697 idx++;
698 }
699 if (err) {
700 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
701 engine->name, err, reg);
702
703 if (ro_reg)
704 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
705 engine->name, reg, results[0]);
706 else
707 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
708 engine->name, reg, results[0], rsvd);
709
710 expect = results[0];
711 idx = 1;
712 for (v = 0; v < ARRAY_SIZE(values); v++) {
713 u32 w = values[v];
714
715 if (ro_reg)
716 expect = results[0];
717 else
718 expect = reg_write(expect, w, rsvd);
719 pr_info("Wrote %08x, read %08x, expect %08x\n",
720 w, results[idx], expect);
721 idx++;
722 }
723 for (v = 0; v < ARRAY_SIZE(values); v++) {
724 u32 w = ~values[v];
725
726 if (ro_reg)
727 expect = results[0];
728 else
729 expect = reg_write(expect, w, rsvd);
730 pr_info("Wrote %08x, read %08x, expect %08x\n",
731 w, results[idx], expect);
732 idx++;
733 }
734
735 err = -EINVAL;
736 }
737out_unmap_scratch:
738 i915_gem_object_unpin_map(scratch->obj);
739out_unmap_batch:
740 if (cs)
741 i915_gem_object_unpin_map(batch->obj);
742out_ctx:
743 intel_context_unpin(ce);
744out:
745 if (err == -EDEADLK) {
746 err = i915_gem_ww_ctx_backoff(&ww);
747 if (!err)
748 goto retry;
749 }
750 i915_gem_ww_ctx_fini(&ww);
751 if (err)
752 break;
753 }
754
755 if (igt_flush_test(engine->i915))
756 err = -EIO;
757
758 i915_vma_unpin_and_release(&batch, 0);
759out_scratch:
760 i915_vma_unpin_and_release(&scratch, 0);
761 return err;
762}
763
764static int live_dirty_whitelist(void *arg)
765{
766 struct intel_gt *gt = arg;
767 struct intel_engine_cs *engine;
768 enum intel_engine_id id;
769
770
771
772 if (GRAPHICS_VER(gt->i915) < 7)
773 return 0;
774
775 for_each_engine(engine, gt, id) {
776 struct intel_context *ce;
777 int err;
778
779 if (engine->whitelist.count == 0)
780 continue;
781
782 ce = intel_context_create(engine);
783 if (IS_ERR(ce))
784 return PTR_ERR(ce);
785
786 err = check_dirty_whitelist(ce);
787 intel_context_put(ce);
788 if (err)
789 return err;
790 }
791
792 return 0;
793}
794
795static int live_reset_whitelist(void *arg)
796{
797 struct intel_gt *gt = arg;
798 struct intel_engine_cs *engine;
799 enum intel_engine_id id;
800 int err = 0;
801
802
803 igt_global_reset_lock(gt);
804
805 for_each_engine(engine, gt, id) {
806 if (engine->whitelist.count == 0)
807 continue;
808
809 if (intel_has_reset_engine(gt)) {
810 if (intel_engine_uses_guc(engine)) {
811 struct intel_selftest_saved_policy saved;
812 int err2;
813
814 err = intel_selftest_modify_policy(engine, &saved,
815 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
816 if (err)
817 goto out;
818
819 err = check_whitelist_across_reset(engine,
820 do_guc_reset,
821 "guc");
822
823 err2 = intel_selftest_restore_policy(engine, &saved);
824 if (err == 0)
825 err = err2;
826 } else {
827 err = check_whitelist_across_reset(engine,
828 do_engine_reset,
829 "engine");
830 }
831
832 if (err)
833 goto out;
834 }
835
836 if (intel_has_gpu_reset(gt)) {
837 err = check_whitelist_across_reset(engine,
838 do_device_reset,
839 "device");
840 if (err)
841 goto out;
842 }
843 }
844
845out:
846 igt_global_reset_unlock(gt);
847 return err;
848}
849
850static int read_whitelisted_registers(struct intel_context *ce,
851 struct i915_vma *results)
852{
853 struct intel_engine_cs *engine = ce->engine;
854 struct i915_request *rq;
855 int i, err = 0;
856 u32 srm, *cs;
857
858 rq = intel_context_create_request(ce);
859 if (IS_ERR(rq))
860 return PTR_ERR(rq);
861
862 i915_vma_lock(results);
863 err = i915_request_await_object(rq, results->obj, true);
864 if (err == 0)
865 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
866 i915_vma_unlock(results);
867 if (err)
868 goto err_req;
869
870 srm = MI_STORE_REGISTER_MEM;
871 if (GRAPHICS_VER(engine->i915) >= 8)
872 srm++;
873
874 cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
875 if (IS_ERR(cs)) {
876 err = PTR_ERR(cs);
877 goto err_req;
878 }
879
880 for (i = 0; i < engine->whitelist.count; i++) {
881 u64 offset = results->node.start + sizeof(u32) * i;
882 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
883
884
885 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
886
887 *cs++ = srm;
888 *cs++ = reg;
889 *cs++ = lower_32_bits(offset);
890 *cs++ = upper_32_bits(offset);
891 }
892 intel_ring_advance(rq, cs);
893
894err_req:
895 return request_add_sync(rq, err);
896}
897
898static int scrub_whitelisted_registers(struct intel_context *ce)
899{
900 struct intel_engine_cs *engine = ce->engine;
901 struct i915_request *rq;
902 struct i915_vma *batch;
903 int i, err = 0;
904 u32 *cs;
905
906 batch = create_batch(ce->vm);
907 if (IS_ERR(batch))
908 return PTR_ERR(batch);
909
910 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
911 if (IS_ERR(cs)) {
912 err = PTR_ERR(cs);
913 goto err_batch;
914 }
915
916 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
917 for (i = 0; i < engine->whitelist.count; i++) {
918 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
919
920 if (ro_register(reg))
921 continue;
922
923
924 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
925
926 *cs++ = reg;
927 *cs++ = 0xffffffff;
928 }
929 *cs++ = MI_BATCH_BUFFER_END;
930
931 i915_gem_object_flush_map(batch->obj);
932 intel_gt_chipset_flush(engine->gt);
933
934 rq = intel_context_create_request(ce);
935 if (IS_ERR(rq)) {
936 err = PTR_ERR(rq);
937 goto err_unpin;
938 }
939
940 if (engine->emit_init_breadcrumb) {
941 err = engine->emit_init_breadcrumb(rq);
942 if (err)
943 goto err_request;
944 }
945
946 i915_vma_lock(batch);
947 err = i915_request_await_object(rq, batch->obj, false);
948 if (err == 0)
949 err = i915_vma_move_to_active(batch, rq, 0);
950 i915_vma_unlock(batch);
951 if (err)
952 goto err_request;
953
954
955 err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
956
957err_request:
958 err = request_add_sync(rq, err);
959
960err_unpin:
961 i915_gem_object_unpin_map(batch->obj);
962err_batch:
963 i915_vma_unpin_and_release(&batch, 0);
964 return err;
965}
966
967struct regmask {
968 i915_reg_t reg;
969 u8 graphics_ver;
970};
971
972static bool find_reg(struct drm_i915_private *i915,
973 i915_reg_t reg,
974 const struct regmask *tbl,
975 unsigned long count)
976{
977 u32 offset = i915_mmio_reg_offset(reg);
978
979 while (count--) {
980 if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
981 i915_mmio_reg_offset(tbl->reg) == offset)
982 return true;
983 tbl++;
984 }
985
986 return false;
987}
988
989static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
990{
991
992 static const struct regmask pardon[] = {
993 { GEN9_CTX_PREEMPT_REG, 9 },
994 { GEN8_L3SQCREG4, 9 },
995 };
996
997 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
998}
999
1000static bool result_eq(struct intel_engine_cs *engine,
1001 u32 a, u32 b, i915_reg_t reg)
1002{
1003 if (a != b && !pardon_reg(engine->i915, reg)) {
1004 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
1005 i915_mmio_reg_offset(reg), a, b);
1006 return false;
1007 }
1008
1009 return true;
1010}
1011
1012static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
1013{
1014
1015 static const struct regmask wo[] = {
1016 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
1017 };
1018
1019 return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
1020}
1021
1022static bool result_neq(struct intel_engine_cs *engine,
1023 u32 a, u32 b, i915_reg_t reg)
1024{
1025 if (a == b && !writeonly_reg(engine->i915, reg)) {
1026 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
1027 i915_mmio_reg_offset(reg), a);
1028 return false;
1029 }
1030
1031 return true;
1032}
1033
1034static int
1035check_whitelisted_registers(struct intel_engine_cs *engine,
1036 struct i915_vma *A,
1037 struct i915_vma *B,
1038 bool (*fn)(struct intel_engine_cs *engine,
1039 u32 a, u32 b,
1040 i915_reg_t reg))
1041{
1042 u32 *a, *b;
1043 int i, err;
1044
1045 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1046 if (IS_ERR(a))
1047 return PTR_ERR(a);
1048
1049 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1050 if (IS_ERR(b)) {
1051 err = PTR_ERR(b);
1052 goto err_a;
1053 }
1054
1055 err = 0;
1056 for (i = 0; i < engine->whitelist.count; i++) {
1057 const struct i915_wa *wa = &engine->whitelist.list[i];
1058
1059 if (i915_mmio_reg_offset(wa->reg) &
1060 RING_FORCE_TO_NONPRIV_ACCESS_RD)
1061 continue;
1062
1063 if (!fn(engine, a[i], b[i], wa->reg))
1064 err = -EINVAL;
1065 }
1066
1067 i915_gem_object_unpin_map(B->obj);
1068err_a:
1069 i915_gem_object_unpin_map(A->obj);
1070 return err;
1071}
1072
1073static int live_isolated_whitelist(void *arg)
1074{
1075 struct intel_gt *gt = arg;
1076 struct {
1077 struct i915_vma *scratch[2];
1078 } client[2] = {};
1079 struct intel_engine_cs *engine;
1080 enum intel_engine_id id;
1081 int i, err = 0;
1082
1083
1084
1085
1086
1087
1088 if (!intel_engines_has_context_isolation(gt->i915))
1089 return 0;
1090
1091 for (i = 0; i < ARRAY_SIZE(client); i++) {
1092 client[i].scratch[0] =
1093 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1094 if (IS_ERR(client[i].scratch[0])) {
1095 err = PTR_ERR(client[i].scratch[0]);
1096 goto err;
1097 }
1098
1099 client[i].scratch[1] =
1100 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1101 if (IS_ERR(client[i].scratch[1])) {
1102 err = PTR_ERR(client[i].scratch[1]);
1103 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1104 goto err;
1105 }
1106 }
1107
1108 for_each_engine(engine, gt, id) {
1109 struct intel_context *ce[2];
1110
1111 if (!engine->kernel_context->vm)
1112 continue;
1113
1114 if (!whitelist_writable_count(engine))
1115 continue;
1116
1117 ce[0] = intel_context_create(engine);
1118 if (IS_ERR(ce[0])) {
1119 err = PTR_ERR(ce[0]);
1120 break;
1121 }
1122 ce[1] = intel_context_create(engine);
1123 if (IS_ERR(ce[1])) {
1124 err = PTR_ERR(ce[1]);
1125 intel_context_put(ce[0]);
1126 break;
1127 }
1128
1129
1130 err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1131 if (err)
1132 goto err_ce;
1133
1134
1135 err = scrub_whitelisted_registers(ce[0]);
1136 if (err)
1137 goto err_ce;
1138
1139
1140 err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1141 if (err)
1142 goto err_ce;
1143
1144
1145 err = check_whitelisted_registers(engine,
1146 client[0].scratch[0],
1147 client[1].scratch[0],
1148 result_eq);
1149 if (err)
1150 goto err_ce;
1151
1152
1153 err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1154 if (err)
1155 goto err_ce;
1156
1157
1158 err = check_whitelisted_registers(engine,
1159 client[0].scratch[0],
1160 client[0].scratch[1],
1161 result_neq);
1162err_ce:
1163 intel_context_put(ce[1]);
1164 intel_context_put(ce[0]);
1165 if (err)
1166 break;
1167 }
1168
1169err:
1170 for (i = 0; i < ARRAY_SIZE(client); i++) {
1171 i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1172 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1173 }
1174
1175 if (igt_flush_test(gt->i915))
1176 err = -EIO;
1177
1178 return err;
1179}
1180
1181static bool
1182verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1183 const char *str)
1184{
1185 struct intel_engine_cs *engine;
1186 enum intel_engine_id id;
1187 bool ok = true;
1188
1189 ok &= wa_list_verify(gt, &lists->gt_wa_list, str);
1190
1191 for_each_engine(engine, gt, id) {
1192 struct intel_context *ce;
1193
1194 ce = intel_context_create(engine);
1195 if (IS_ERR(ce))
1196 return false;
1197
1198 ok &= engine_wa_list_verify(ce,
1199 &lists->engine[id].wa_list,
1200 str) == 0;
1201
1202 ok &= engine_wa_list_verify(ce,
1203 &lists->engine[id].ctx_wa_list,
1204 str) == 0;
1205
1206 intel_context_put(ce);
1207 }
1208
1209 return ok;
1210}
1211
1212static int
1213live_gpu_reset_workarounds(void *arg)
1214{
1215 struct intel_gt *gt = arg;
1216 intel_wakeref_t wakeref;
1217 struct wa_lists *lists;
1218 bool ok;
1219
1220 if (!intel_has_gpu_reset(gt))
1221 return 0;
1222
1223 lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1224 if (!lists)
1225 return -ENOMEM;
1226
1227 pr_info("Verifying after GPU reset...\n");
1228
1229 igt_global_reset_lock(gt);
1230 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1231
1232 reference_lists_init(gt, lists);
1233
1234 ok = verify_wa_lists(gt, lists, "before reset");
1235 if (!ok)
1236 goto out;
1237
1238 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1239
1240 ok = verify_wa_lists(gt, lists, "after reset");
1241
1242out:
1243 reference_lists_fini(gt, lists);
1244 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1245 igt_global_reset_unlock(gt);
1246 kfree(lists);
1247
1248 return ok ? 0 : -ESRCH;
1249}
1250
1251static int
1252live_engine_reset_workarounds(void *arg)
1253{
1254 struct intel_gt *gt = arg;
1255 struct intel_engine_cs *engine;
1256 enum intel_engine_id id;
1257 struct intel_context *ce;
1258 struct igt_spinner spin;
1259 struct i915_request *rq;
1260 intel_wakeref_t wakeref;
1261 struct wa_lists *lists;
1262 int ret = 0;
1263
1264 if (!intel_has_reset_engine(gt))
1265 return 0;
1266
1267 lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1268 if (!lists)
1269 return -ENOMEM;
1270
1271 igt_global_reset_lock(gt);
1272 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1273
1274 reference_lists_init(gt, lists);
1275
1276 for_each_engine(engine, gt, id) {
1277 struct intel_selftest_saved_policy saved;
1278 bool using_guc = intel_engine_uses_guc(engine);
1279 bool ok;
1280 int ret2;
1281
1282 pr_info("Verifying after %s reset...\n", engine->name);
1283 ret = intel_selftest_modify_policy(engine, &saved,
1284 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1285 if (ret)
1286 break;
1287
1288 ce = intel_context_create(engine);
1289 if (IS_ERR(ce)) {
1290 ret = PTR_ERR(ce);
1291 goto restore;
1292 }
1293
1294 if (!using_guc) {
1295 ok = verify_wa_lists(gt, lists, "before reset");
1296 if (!ok) {
1297 ret = -ESRCH;
1298 goto err;
1299 }
1300
1301 ret = intel_engine_reset(engine, "live_workarounds:idle");
1302 if (ret) {
1303 pr_err("%s: Reset failed while idle\n", engine->name);
1304 goto err;
1305 }
1306
1307 ok = verify_wa_lists(gt, lists, "after idle reset");
1308 if (!ok) {
1309 ret = -ESRCH;
1310 goto err;
1311 }
1312 }
1313
1314 ret = igt_spinner_init(&spin, engine->gt);
1315 if (ret)
1316 goto err;
1317
1318 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1319 if (IS_ERR(rq)) {
1320 ret = PTR_ERR(rq);
1321 igt_spinner_fini(&spin);
1322 goto err;
1323 }
1324
1325 ret = request_add_spin(rq, &spin);
1326 if (ret) {
1327 pr_err("%s: Spinner failed to start\n", engine->name);
1328 igt_spinner_fini(&spin);
1329 goto err;
1330 }
1331
1332
1333 if (i915_request_completed(rq)) {
1334 ret = -ETIMEDOUT;
1335 goto skip;
1336 }
1337
1338 if (!using_guc) {
1339 ret = intel_engine_reset(engine, "live_workarounds:active");
1340 if (ret) {
1341 pr_err("%s: Reset failed on an active spinner\n",
1342 engine->name);
1343 igt_spinner_fini(&spin);
1344 goto err;
1345 }
1346 }
1347
1348
1349 if (ret == 0)
1350 ret = intel_selftest_wait_for_rq(rq);
1351
1352skip:
1353 igt_spinner_end(&spin);
1354 igt_spinner_fini(&spin);
1355
1356 ok = verify_wa_lists(gt, lists, "after busy reset");
1357 if (!ok)
1358 ret = -ESRCH;
1359
1360err:
1361 intel_context_put(ce);
1362
1363restore:
1364 ret2 = intel_selftest_restore_policy(engine, &saved);
1365 if (ret == 0)
1366 ret = ret2;
1367 if (ret)
1368 break;
1369 }
1370
1371 reference_lists_fini(gt, lists);
1372 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1373 igt_global_reset_unlock(gt);
1374 kfree(lists);
1375
1376 igt_flush_test(gt->i915);
1377
1378 return ret;
1379}
1380
1381int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1382{
1383 static const struct i915_subtest tests[] = {
1384 SUBTEST(live_dirty_whitelist),
1385 SUBTEST(live_reset_whitelist),
1386 SUBTEST(live_isolated_whitelist),
1387 SUBTEST(live_gpu_reset_workarounds),
1388 SUBTEST(live_engine_reset_workarounds),
1389 };
1390
1391 if (intel_gt_is_wedged(to_gt(i915)))
1392 return 0;
1393
1394 return intel_gt_live_subtests(tests, to_gt(i915));
1395}
1396
