1
2
3
4
5
6#include "gem/i915_gem_pm.h"
7#include "gt/intel_engine_user.h"
8#include "gt/intel_gt.h"
9#include "i915_selftest.h"
10#include "intel_reset.h"
11
12#include "selftests/igt_flush_test.h"
13#include "selftests/igt_reset.h"
14#include "selftests/igt_spinner.h"
15#include "selftests/intel_scheduler_helpers.h"
16#include "selftests/mock_drm.h"
17
18#include "gem/selftests/igt_gem_utils.h"
19#include "gem/selftests/mock_context.h"
20
21static const struct wo_register {
22 enum intel_platform platform;
23 u32 reg;
24} wo_registers[] = {
25 { INTEL_GEMINILAKE, 0x731c }
26};
27
28struct wa_lists {
29 struct i915_wa_list gt_wa_list;
30 struct {
31 struct i915_wa_list wa_list;
32 struct i915_wa_list ctx_wa_list;
33 } engine[I915_NUM_ENGINES];
34};
35
36static int request_add_sync(struct i915_request *rq, int err)
37{
38 i915_request_get(rq);
39 i915_request_add(rq);
40 if (i915_request_wait(rq, 0, HZ / 5) < 0)
41 err = -EIO;
42 i915_request_put(rq);
43
44 return err;
45}
46
47static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
48{
49 int err = 0;
50
51 i915_request_get(rq);
52 i915_request_add(rq);
53 if (spin && !igt_wait_for_spinner(spin, rq))
54 err = -ETIMEDOUT;
55 i915_request_put(rq);
56
57 return err;
58}
59
60static void
61reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
62{
63 struct intel_engine_cs *engine;
64 enum intel_engine_id id;
65
66 memset(lists, 0, sizeof(*lists));
67
68 wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
69 gt_init_workarounds(gt, &lists->gt_wa_list);
70 wa_init_finish(&lists->gt_wa_list);
71
72 for_each_engine(engine, gt, id) {
73 struct i915_wa_list *wal = &lists->engine[id].wa_list;
74
75 wa_init_start(wal, "REF", engine->name);
76 engine_init_workarounds(engine, wal);
77 wa_init_finish(wal);
78
79 __intel_engine_init_ctx_wa(engine,
80 &lists->engine[id].ctx_wa_list,
81 "CTX_REF");
82 }
83}
84
85static void
86reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
87{
88 struct intel_engine_cs *engine;
89 enum intel_engine_id id;
90
91 for_each_engine(engine, gt, id)
92 intel_wa_list_free(&lists->engine[id].wa_list);
93
94 intel_wa_list_free(&lists->gt_wa_list);
95}
96
97static struct drm_i915_gem_object *
98read_nonprivs(struct intel_context *ce)
99{
100 struct intel_engine_cs *engine = ce->engine;
101 const u32 base = engine->mmio_base;
102 struct drm_i915_gem_object *result;
103 struct i915_request *rq;
104 struct i915_vma *vma;
105 u32 srm, *cs;
106 int err;
107 int i;
108
109 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
110 if (IS_ERR(result))
111 return result;
112
113 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
114
115 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
116 if (IS_ERR(cs)) {
117 err = PTR_ERR(cs);
118 goto err_obj;
119 }
120 memset(cs, 0xc5, PAGE_SIZE);
121 i915_gem_object_flush_map(result);
122 i915_gem_object_unpin_map(result);
123
124 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
125 if (IS_ERR(vma)) {
126 err = PTR_ERR(vma);
127 goto err_obj;
128 }
129
130 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
131 if (err)
132 goto err_obj;
133
134 rq = intel_context_create_request(ce);
135 if (IS_ERR(rq)) {
136 err = PTR_ERR(rq);
137 goto err_pin;
138 }
139
140 i915_vma_lock(vma);
141 err = i915_request_await_object(rq, vma->obj, true);
142 if (err == 0)
143 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
144 i915_vma_unlock(vma);
145 if (err)
146 goto err_req;
147
148 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
149 if (GRAPHICS_VER(engine->i915) >= 8)
150 srm++;
151
152 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
153 if (IS_ERR(cs)) {
154 err = PTR_ERR(cs);
155 goto err_req;
156 }
157
158 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
159 *cs++ = srm;
160 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
161 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
162 *cs++ = 0;
163 }
164 intel_ring_advance(rq, cs);
165
166 i915_request_add(rq);
167 i915_vma_unpin(vma);
168
169 return result;
170
171err_req:
172 i915_request_add(rq);
173err_pin:
174 i915_vma_unpin(vma);
175err_obj:
176 i915_gem_object_put(result);
177 return ERR_PTR(err);
178}
179
180static u32
181get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
182{
183 i915_reg_t reg = i < engine->whitelist.count ?
184 engine->whitelist.list[i].reg :
185 RING_NOPID(engine->mmio_base);
186
187 return i915_mmio_reg_offset(reg);
188}
189
190static void
191print_results(const struct intel_engine_cs *engine, const u32 *results)
192{
193 unsigned int i;
194
195 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
196 u32 expected = get_whitelist_reg(engine, i);
197 u32 actual = results[i];
198
199 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
200 i, expected, actual);
201 }
202}
203
204static int check_whitelist(struct intel_context *ce)
205{
206 struct intel_engine_cs *engine = ce->engine;
207 struct drm_i915_gem_object *results;
208 struct intel_wedge_me wedge;
209 u32 *vaddr;
210 int err;
211 int i;
212
213 results = read_nonprivs(ce);
214 if (IS_ERR(results))
215 return PTR_ERR(results);
216
217 err = 0;
218 i915_gem_object_lock(results, NULL);
219 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5)
220 err = i915_gem_object_set_to_cpu_domain(results, false);
221
222 if (intel_gt_is_wedged(engine->gt))
223 err = -EIO;
224 if (err)
225 goto out_put;
226
227 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
228 if (IS_ERR(vaddr)) {
229 err = PTR_ERR(vaddr);
230 goto out_put;
231 }
232
233 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
234 u32 expected = get_whitelist_reg(engine, i);
235 u32 actual = vaddr[i];
236
237 if (expected != actual) {
238 print_results(engine, vaddr);
239 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
240 i, expected, actual);
241
242 err = -EINVAL;
243 break;
244 }
245 }
246
247 i915_gem_object_unpin_map(results);
248out_put:
249 i915_gem_object_unlock(results);
250 i915_gem_object_put(results);
251 return err;
252}
253
254static int do_device_reset(struct intel_engine_cs *engine)
255{
256 intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
257 return 0;
258}
259
260static int do_engine_reset(struct intel_engine_cs *engine)
261{
262 return intel_engine_reset(engine, "live_workarounds");
263}
264
265static int do_guc_reset(struct intel_engine_cs *engine)
266{
267
268 return 0;
269}
270
271static int
272switch_to_scratch_context(struct intel_engine_cs *engine,
273 struct igt_spinner *spin,
274 struct i915_request **rq)
275{
276 struct intel_context *ce;
277 int err = 0;
278
279 ce = intel_context_create(engine);
280 if (IS_ERR(ce))
281 return PTR_ERR(ce);
282
283 *rq = igt_spinner_create_request(spin, ce, MI_NOOP);
284 intel_context_put(ce);
285
286 if (IS_ERR(*rq)) {
287 spin = NULL;
288 err = PTR_ERR(*rq);
289 goto err;
290 }
291
292 err = request_add_spin(*rq, spin);
293err:
294 if (err && spin)
295 igt_spinner_end(spin);
296
297 return err;
298}
299
300static int check_whitelist_across_reset(struct intel_engine_cs *engine,
301 int (*reset)(struct intel_engine_cs *),
302 const char *name)
303{
304 struct intel_context *ce, *tmp;
305 struct igt_spinner spin;
306 struct i915_request *rq;
307 intel_wakeref_t wakeref;
308 int err;
309
310 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
311 engine->whitelist.count, engine->name, name);
312
313 ce = intel_context_create(engine);
314 if (IS_ERR(ce))
315 return PTR_ERR(ce);
316
317 err = igt_spinner_init(&spin, engine->gt);
318 if (err)
319 goto out_ctx;
320
321 err = check_whitelist(ce);
322 if (err) {
323 pr_err("Invalid whitelist *before* %s reset!\n", name);
324 goto out_spin;
325 }
326
327 err = switch_to_scratch_context(engine, &spin, &rq);
328 if (err)
329 goto out_spin;
330
331
332 if (i915_request_completed(rq)) {
333 pr_err("%s spinner failed to start\n", name);
334 err = -ETIMEDOUT;
335 goto out_spin;
336 }
337
338 with_intel_runtime_pm(engine->uncore->rpm, wakeref)
339 err = reset(engine);
340
341
342 if (err == 0)
343 err = intel_selftest_wait_for_rq(rq);
344
345 igt_spinner_end(&spin);
346
347 if (err) {
348 pr_err("%s reset failed\n", name);
349 goto out_spin;
350 }
351
352 err = check_whitelist(ce);
353 if (err) {
354 pr_err("Whitelist not preserved in context across %s reset!\n",
355 name);
356 goto out_spin;
357 }
358
359 tmp = intel_context_create(engine);
360 if (IS_ERR(tmp)) {
361 err = PTR_ERR(tmp);
362 goto out_spin;
363 }
364 intel_context_put(ce);
365 ce = tmp;
366
367 err = check_whitelist(ce);
368 if (err) {
369 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
370 name);
371 goto out_spin;
372 }
373
374out_spin:
375 igt_spinner_fini(&spin);
376out_ctx:
377 intel_context_put(ce);
378 return err;
379}
380
381static struct i915_vma *create_batch(struct i915_address_space *vm)
382{
383 struct drm_i915_gem_object *obj;
384 struct i915_vma *vma;
385 int err;
386
387 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
388 if (IS_ERR(obj))
389 return ERR_CAST(obj);
390
391 vma = i915_vma_instance(obj, vm, NULL);
392 if (IS_ERR(vma)) {
393 err = PTR_ERR(vma);
394 goto err_obj;
395 }
396
397 err = i915_vma_pin(vma, 0, 0, PIN_USER);
398 if (err)
399 goto err_obj;
400
401 return vma;
402
403err_obj:
404 i915_gem_object_put(obj);
405 return ERR_PTR(err);
406}
407
408static u32 reg_write(u32 old, u32 new, u32 rsvd)
409{
410 if (rsvd == 0x0000ffff) {
411 old &= ~(new >> 16);
412 old |= new & (new >> 16);
413 } else {
414 old &= ~rsvd;
415 old |= new & rsvd;
416 }
417
418 return old;
419}
420
421static bool wo_register(struct intel_engine_cs *engine, u32 reg)
422{
423 enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
424 int i;
425
426 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
427 RING_FORCE_TO_NONPRIV_ACCESS_WR)
428 return true;
429
430 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
431 if (wo_registers[i].platform == platform &&
432 wo_registers[i].reg == reg)
433 return true;
434 }
435
436 return false;
437}
438
439static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
440{
441 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
442 switch (reg) {
443 case 0x358:
444 case 0x35c:
445 case 0x3a8:
446 return true;
447
448 default:
449 return false;
450 }
451}
452
453static bool ro_register(u32 reg)
454{
455 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
456 RING_FORCE_TO_NONPRIV_ACCESS_RD)
457 return true;
458
459 return false;
460}
461
462static int whitelist_writable_count(struct intel_engine_cs *engine)
463{
464 int count = engine->whitelist.count;
465 int i;
466
467 for (i = 0; i < engine->whitelist.count; i++) {
468 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
469
470 if (ro_register(reg))
471 count--;
472 }
473
474 return count;
475}
476
477static int check_dirty_whitelist(struct intel_context *ce)
478{
479 const u32 values[] = {
480 0x00000000,
481 0x01010101,
482 0x10100101,
483 0x03030303,
484 0x30300303,
485 0x05050505,
486 0x50500505,
487 0x0f0f0f0f,
488 0xf00ff00f,
489 0x10101010,
490 0xf0f01010,
491 0x30303030,
492 0xa0a03030,
493 0x50505050,
494 0xc0c05050,
495 0xf0f0f0f0,
496 0x11111111,
497 0x33333333,
498 0x55555555,
499 0x0000ffff,
500 0x00ff00ff,
501 0xff0000ff,
502 0xffff00ff,
503 0xffffffff,
504 };
505 struct intel_engine_cs *engine = ce->engine;
506 struct i915_vma *scratch;
507 struct i915_vma *batch;
508 int err = 0, i, v, sz;
509 u32 *cs, *results;
510
511 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
512 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
513 if (IS_ERR(scratch))
514 return PTR_ERR(scratch);
515
516 batch = create_batch(ce->vm);
517 if (IS_ERR(batch)) {
518 err = PTR_ERR(batch);
519 goto out_scratch;
520 }
521
522 for (i = 0; i < engine->whitelist.count; i++) {
523 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
524 struct i915_gem_ww_ctx ww;
525 u64 addr = scratch->node.start;
526 struct i915_request *rq;
527 u32 srm, lrm, rsvd;
528 u32 expect;
529 int idx;
530 bool ro_reg;
531
532 if (wo_register(engine, reg))
533 continue;
534
535 if (timestamp(engine, reg))
536 continue;
537
538 ro_reg = ro_register(reg);
539
540 i915_gem_ww_ctx_init(&ww, false);
541retry:
542 cs = NULL;
543 err = i915_gem_object_lock(scratch->obj, &ww);
544 if (!err)
545 err = i915_gem_object_lock(batch->obj, &ww);
546 if (!err)
547 err = intel_context_pin_ww(ce, &ww);
548 if (err)
549 goto out;
550
551 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
552 if (IS_ERR(cs)) {
553 err = PTR_ERR(cs);
554 goto out_ctx;
555 }
556
557 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
558 if (IS_ERR(results)) {
559 err = PTR_ERR(results);
560 goto out_unmap_batch;
561 }
562
563
564 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
565
566 srm = MI_STORE_REGISTER_MEM;
567 lrm = MI_LOAD_REGISTER_MEM;
568 if (GRAPHICS_VER(engine->i915) >= 8)
569 lrm++, srm++;
570
571 pr_debug("%s: Writing garbage to %x\n",
572 engine->name, reg);
573
574
575 *cs++ = srm;
576 *cs++ = reg;
577 *cs++ = lower_32_bits(addr);
578 *cs++ = upper_32_bits(addr);
579
580 idx = 1;
581 for (v = 0; v < ARRAY_SIZE(values); v++) {
582
583 *cs++ = MI_LOAD_REGISTER_IMM(1);
584 *cs++ = reg;
585 *cs++ = values[v];
586
587
588 *cs++ = srm;
589 *cs++ = reg;
590 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
591 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
592 idx++;
593 }
594 for (v = 0; v < ARRAY_SIZE(values); v++) {
595
596 *cs++ = MI_LOAD_REGISTER_IMM(1);
597 *cs++ = reg;
598 *cs++ = ~values[v];
599
600
601 *cs++ = srm;
602 *cs++ = reg;
603 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
604 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
605 idx++;
606 }
607 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
608
609
610 *cs++ = lrm;
611 *cs++ = reg;
612 *cs++ = lower_32_bits(addr);
613 *cs++ = upper_32_bits(addr);
614
615 *cs++ = MI_BATCH_BUFFER_END;
616
617 i915_gem_object_flush_map(batch->obj);
618 i915_gem_object_unpin_map(batch->obj);
619 intel_gt_chipset_flush(engine->gt);
620 cs = NULL;
621
622 rq = i915_request_create(ce);
623 if (IS_ERR(rq)) {
624 err = PTR_ERR(rq);
625 goto out_unmap_scratch;
626 }
627
628 if (engine->emit_init_breadcrumb) {
629 err = engine->emit_init_breadcrumb(rq);
630 if (err)
631 goto err_request;
632 }
633
634 err = i915_request_await_object(rq, batch->obj, false);
635 if (err == 0)
636 err = i915_vma_move_to_active(batch, rq, 0);
637 if (err)
638 goto err_request;
639
640 err = i915_request_await_object(rq, scratch->obj, true);
641 if (err == 0)
642 err = i915_vma_move_to_active(scratch, rq,
643 EXEC_OBJECT_WRITE);
644 if (err)
645 goto err_request;
646
647 err = engine->emit_bb_start(rq,
648 batch->node.start, PAGE_SIZE,
649 0);
650 if (err)
651 goto err_request;
652
653err_request:
654 err = request_add_sync(rq, err);
655 if (err) {
656 pr_err("%s: Futzing %x timedout; cancelling test\n",
657 engine->name, reg);
658 intel_gt_set_wedged(engine->gt);
659 goto out_unmap_scratch;
660 }
661
662 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
663 if (!ro_reg) {
664
665 rsvd = results[ARRAY_SIZE(values)];
666 if (!rsvd) {
667 pr_err("%s: Unable to write to whitelisted register %x\n",
668 engine->name, reg);
669 err = -EINVAL;
670 goto out_unmap_scratch;
671 }
672 } else {
673 rsvd = 0;
674 }
675
676 expect = results[0];
677 idx = 1;
678 for (v = 0; v < ARRAY_SIZE(values); v++) {
679 if (ro_reg)
680 expect = results[0];
681 else
682 expect = reg_write(expect, values[v], rsvd);
683
684 if (results[idx] != expect)
685 err++;
686 idx++;
687 }
688 for (v = 0; v < ARRAY_SIZE(values); v++) {
689 if (ro_reg)
690 expect = results[0];
691 else
692 expect = reg_write(expect, ~values[v], rsvd);
693
694 if (results[idx] != expect)
695 err++;
696 idx++;
697 }
698 if (err) {
699 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
700 engine->name, err, reg);
701
702 if (ro_reg)
703 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
704 engine->name, reg, results[0]);
705 else
706 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
707 engine->name, reg, results[0], rsvd);
708
709 expect = results[0];
710 idx = 1;
711 for (v = 0; v < ARRAY_SIZE(values); v++) {
712 u32 w = values[v];
713
714 if (ro_reg)
715 expect = results[0];
716 else
717 expect = reg_write(expect, w, rsvd);
718 pr_info("Wrote %08x, read %08x, expect %08x\n",
719 w, results[idx], expect);
720 idx++;
721 }
722 for (v = 0; v < ARRAY_SIZE(values); v++) {
723 u32 w = ~values[v];
724
725 if (ro_reg)
726 expect = results[0];
727 else
728 expect = reg_write(expect, w, rsvd);
729 pr_info("Wrote %08x, read %08x, expect %08x\n",
730 w, results[idx], expect);
731 idx++;
732 }
733
734 err = -EINVAL;
735 }
736out_unmap_scratch:
737 i915_gem_object_unpin_map(scratch->obj);
738out_unmap_batch:
739 if (cs)
740 i915_gem_object_unpin_map(batch->obj);
741out_ctx:
742 intel_context_unpin(ce);
743out:
744 if (err == -EDEADLK) {
745 err = i915_gem_ww_ctx_backoff(&ww);
746 if (!err)
747 goto retry;
748 }
749 i915_gem_ww_ctx_fini(&ww);
750 if (err)
751 break;
752 }
753
754 if (igt_flush_test(engine->i915))
755 err = -EIO;
756
757 i915_vma_unpin_and_release(&batch, 0);
758out_scratch:
759 i915_vma_unpin_and_release(&scratch, 0);
760 return err;
761}
762
763static int live_dirty_whitelist(void *arg)
764{
765 struct intel_gt *gt = arg;
766 struct intel_engine_cs *engine;
767 enum intel_engine_id id;
768
769
770
771 if (GRAPHICS_VER(gt->i915) < 7)
772 return 0;
773
774 for_each_engine(engine, gt, id) {
775 struct intel_context *ce;
776 int err;
777
778 if (engine->whitelist.count == 0)
779 continue;
780
781 ce = intel_context_create(engine);
782 if (IS_ERR(ce))
783 return PTR_ERR(ce);
784
785 err = check_dirty_whitelist(ce);
786 intel_context_put(ce);
787 if (err)
788 return err;
789 }
790
791 return 0;
792}
793
794static int live_reset_whitelist(void *arg)
795{
796 struct intel_gt *gt = arg;
797 struct intel_engine_cs *engine;
798 enum intel_engine_id id;
799 int err = 0;
800
801
802 igt_global_reset_lock(gt);
803
804 for_each_engine(engine, gt, id) {
805 if (engine->whitelist.count == 0)
806 continue;
807
808 if (intel_has_reset_engine(gt)) {
809 if (intel_engine_uses_guc(engine)) {
810 struct intel_selftest_saved_policy saved;
811 int err2;
812
813 err = intel_selftest_modify_policy(engine, &saved,
814 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
815 if (err)
816 goto out;
817
818 err = check_whitelist_across_reset(engine,
819 do_guc_reset,
820 "guc");
821
822 err2 = intel_selftest_restore_policy(engine, &saved);
823 if (err == 0)
824 err = err2;
825 } else {
826 err = check_whitelist_across_reset(engine,
827 do_engine_reset,
828 "engine");
829 }
830
831 if (err)
832 goto out;
833 }
834
835 if (intel_has_gpu_reset(gt)) {
836 err = check_whitelist_across_reset(engine,
837 do_device_reset,
838 "device");
839 if (err)
840 goto out;
841 }
842 }
843
844out:
845 igt_global_reset_unlock(gt);
846 return err;
847}
848
849static int read_whitelisted_registers(struct intel_context *ce,
850 struct i915_vma *results)
851{
852 struct intel_engine_cs *engine = ce->engine;
853 struct i915_request *rq;
854 int i, err = 0;
855 u32 srm, *cs;
856
857 rq = intel_context_create_request(ce);
858 if (IS_ERR(rq))
859 return PTR_ERR(rq);
860
861 i915_vma_lock(results);
862 err = i915_request_await_object(rq, results->obj, true);
863 if (err == 0)
864 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
865 i915_vma_unlock(results);
866 if (err)
867 goto err_req;
868
869 srm = MI_STORE_REGISTER_MEM;
870 if (GRAPHICS_VER(engine->i915) >= 8)
871 srm++;
872
873 cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
874 if (IS_ERR(cs)) {
875 err = PTR_ERR(cs);
876 goto err_req;
877 }
878
879 for (i = 0; i < engine->whitelist.count; i++) {
880 u64 offset = results->node.start + sizeof(u32) * i;
881 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
882
883
884 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
885
886 *cs++ = srm;
887 *cs++ = reg;
888 *cs++ = lower_32_bits(offset);
889 *cs++ = upper_32_bits(offset);
890 }
891 intel_ring_advance(rq, cs);
892
893err_req:
894 return request_add_sync(rq, err);
895}
896
897static int scrub_whitelisted_registers(struct intel_context *ce)
898{
899 struct intel_engine_cs *engine = ce->engine;
900 struct i915_request *rq;
901 struct i915_vma *batch;
902 int i, err = 0;
903 u32 *cs;
904
905 batch = create_batch(ce->vm);
906 if (IS_ERR(batch))
907 return PTR_ERR(batch);
908
909 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
910 if (IS_ERR(cs)) {
911 err = PTR_ERR(cs);
912 goto err_batch;
913 }
914
915 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
916 for (i = 0; i < engine->whitelist.count; i++) {
917 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
918
919 if (ro_register(reg))
920 continue;
921
922
923 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
924
925 *cs++ = reg;
926 *cs++ = 0xffffffff;
927 }
928 *cs++ = MI_BATCH_BUFFER_END;
929
930 i915_gem_object_flush_map(batch->obj);
931 intel_gt_chipset_flush(engine->gt);
932
933 rq = intel_context_create_request(ce);
934 if (IS_ERR(rq)) {
935 err = PTR_ERR(rq);
936 goto err_unpin;
937 }
938
939 if (engine->emit_init_breadcrumb) {
940 err = engine->emit_init_breadcrumb(rq);
941 if (err)
942 goto err_request;
943 }
944
945 i915_vma_lock(batch);
946 err = i915_request_await_object(rq, batch->obj, false);
947 if (err == 0)
948 err = i915_vma_move_to_active(batch, rq, 0);
949 i915_vma_unlock(batch);
950 if (err)
951 goto err_request;
952
953
954 err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
955
956err_request:
957 err = request_add_sync(rq, err);
958
959err_unpin:
960 i915_gem_object_unpin_map(batch->obj);
961err_batch:
962 i915_vma_unpin_and_release(&batch, 0);
963 return err;
964}
965
966struct regmask {
967 i915_reg_t reg;
968 u8 graphics_ver;
969};
970
971static bool find_reg(struct drm_i915_private *i915,
972 i915_reg_t reg,
973 const struct regmask *tbl,
974 unsigned long count)
975{
976 u32 offset = i915_mmio_reg_offset(reg);
977
978 while (count--) {
979 if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
980 i915_mmio_reg_offset(tbl->reg) == offset)
981 return true;
982 tbl++;
983 }
984
985 return false;
986}
987
988static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
989{
990
991 static const struct regmask pardon[] = {
992 { GEN9_CTX_PREEMPT_REG, 9 },
993 { GEN8_L3SQCREG4, 9 },
994 };
995
996 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
997}
998
999static bool result_eq(struct intel_engine_cs *engine,
1000 u32 a, u32 b, i915_reg_t reg)
1001{
1002 if (a != b && !pardon_reg(engine->i915, reg)) {
1003 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
1004 i915_mmio_reg_offset(reg), a, b);
1005 return false;
1006 }
1007
1008 return true;
1009}
1010
1011static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
1012{
1013
1014 static const struct regmask wo[] = {
1015 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
1016 };
1017
1018 return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
1019}
1020
1021static bool result_neq(struct intel_engine_cs *engine,
1022 u32 a, u32 b, i915_reg_t reg)
1023{
1024 if (a == b && !writeonly_reg(engine->i915, reg)) {
1025 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
1026 i915_mmio_reg_offset(reg), a);
1027 return false;
1028 }
1029
1030 return true;
1031}
1032
1033static int
1034check_whitelisted_registers(struct intel_engine_cs *engine,
1035 struct i915_vma *A,
1036 struct i915_vma *B,
1037 bool (*fn)(struct intel_engine_cs *engine,
1038 u32 a, u32 b,
1039 i915_reg_t reg))
1040{
1041 u32 *a, *b;
1042 int i, err;
1043
1044 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1045 if (IS_ERR(a))
1046 return PTR_ERR(a);
1047
1048 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1049 if (IS_ERR(b)) {
1050 err = PTR_ERR(b);
1051 goto err_a;
1052 }
1053
1054 err = 0;
1055 for (i = 0; i < engine->whitelist.count; i++) {
1056 const struct i915_wa *wa = &engine->whitelist.list[i];
1057
1058 if (i915_mmio_reg_offset(wa->reg) &
1059 RING_FORCE_TO_NONPRIV_ACCESS_RD)
1060 continue;
1061
1062 if (!fn(engine, a[i], b[i], wa->reg))
1063 err = -EINVAL;
1064 }
1065
1066 i915_gem_object_unpin_map(B->obj);
1067err_a:
1068 i915_gem_object_unpin_map(A->obj);
1069 return err;
1070}
1071
1072static int live_isolated_whitelist(void *arg)
1073{
1074 struct intel_gt *gt = arg;
1075 struct {
1076 struct i915_vma *scratch[2];
1077 } client[2] = {};
1078 struct intel_engine_cs *engine;
1079 enum intel_engine_id id;
1080 int i, err = 0;
1081
1082
1083
1084
1085
1086
1087 if (!intel_engines_has_context_isolation(gt->i915))
1088 return 0;
1089
1090 for (i = 0; i < ARRAY_SIZE(client); i++) {
1091 client[i].scratch[0] =
1092 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1093 if (IS_ERR(client[i].scratch[0])) {
1094 err = PTR_ERR(client[i].scratch[0]);
1095 goto err;
1096 }
1097
1098 client[i].scratch[1] =
1099 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1100 if (IS_ERR(client[i].scratch[1])) {
1101 err = PTR_ERR(client[i].scratch[1]);
1102 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1103 goto err;
1104 }
1105 }
1106
1107 for_each_engine(engine, gt, id) {
1108 struct intel_context *ce[2];
1109
1110 if (!engine->kernel_context->vm)
1111 continue;
1112
1113 if (!whitelist_writable_count(engine))
1114 continue;
1115
1116 ce[0] = intel_context_create(engine);
1117 if (IS_ERR(ce[0])) {
1118 err = PTR_ERR(ce[0]);
1119 break;
1120 }
1121 ce[1] = intel_context_create(engine);
1122 if (IS_ERR(ce[1])) {
1123 err = PTR_ERR(ce[1]);
1124 intel_context_put(ce[0]);
1125 break;
1126 }
1127
1128
1129 err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1130 if (err)
1131 goto err_ce;
1132
1133
1134 err = scrub_whitelisted_registers(ce[0]);
1135 if (err)
1136 goto err_ce;
1137
1138
1139 err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1140 if (err)
1141 goto err_ce;
1142
1143
1144 err = check_whitelisted_registers(engine,
1145 client[0].scratch[0],
1146 client[1].scratch[0],
1147 result_eq);
1148 if (err)
1149 goto err_ce;
1150
1151
1152 err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1153 if (err)
1154 goto err_ce;
1155
1156
1157 err = check_whitelisted_registers(engine,
1158 client[0].scratch[0],
1159 client[0].scratch[1],
1160 result_neq);
1161err_ce:
1162 intel_context_put(ce[1]);
1163 intel_context_put(ce[0]);
1164 if (err)
1165 break;
1166 }
1167
1168err:
1169 for (i = 0; i < ARRAY_SIZE(client); i++) {
1170 i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1171 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1172 }
1173
1174 if (igt_flush_test(gt->i915))
1175 err = -EIO;
1176
1177 return err;
1178}
1179
1180static bool
1181verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1182 const char *str)
1183{
1184 struct intel_engine_cs *engine;
1185 enum intel_engine_id id;
1186 bool ok = true;
1187
1188 ok &= wa_list_verify(gt, &lists->gt_wa_list, str);
1189
1190 for_each_engine(engine, gt, id) {
1191 struct intel_context *ce;
1192
1193 ce = intel_context_create(engine);
1194 if (IS_ERR(ce))
1195 return false;
1196
1197 ok &= engine_wa_list_verify(ce,
1198 &lists->engine[id].wa_list,
1199 str) == 0;
1200
1201 ok &= engine_wa_list_verify(ce,
1202 &lists->engine[id].ctx_wa_list,
1203 str) == 0;
1204
1205 intel_context_put(ce);
1206 }
1207
1208 return ok;
1209}
1210
1211static int
1212live_gpu_reset_workarounds(void *arg)
1213{
1214 struct intel_gt *gt = arg;
1215 intel_wakeref_t wakeref;
1216 struct wa_lists *lists;
1217 bool ok;
1218
1219 if (!intel_has_gpu_reset(gt))
1220 return 0;
1221
1222 lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1223 if (!lists)
1224 return -ENOMEM;
1225
1226 pr_info("Verifying after GPU reset...\n");
1227
1228 igt_global_reset_lock(gt);
1229 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1230
1231 reference_lists_init(gt, lists);
1232
1233 ok = verify_wa_lists(gt, lists, "before reset");
1234 if (!ok)
1235 goto out;
1236
1237 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1238
1239 ok = verify_wa_lists(gt, lists, "after reset");
1240
1241out:
1242 reference_lists_fini(gt, lists);
1243 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1244 igt_global_reset_unlock(gt);
1245 kfree(lists);
1246
1247 return ok ? 0 : -ESRCH;
1248}
1249
1250static int
1251live_engine_reset_workarounds(void *arg)
1252{
1253 struct intel_gt *gt = arg;
1254 struct intel_engine_cs *engine;
1255 enum intel_engine_id id;
1256 struct intel_context *ce;
1257 struct igt_spinner spin;
1258 struct i915_request *rq;
1259 intel_wakeref_t wakeref;
1260 struct wa_lists *lists;
1261 int ret = 0;
1262
1263 if (!intel_has_reset_engine(gt))
1264 return 0;
1265
1266 lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1267 if (!lists)
1268 return -ENOMEM;
1269
1270 igt_global_reset_lock(gt);
1271 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1272
1273 reference_lists_init(gt, lists);
1274
1275 for_each_engine(engine, gt, id) {
1276 struct intel_selftest_saved_policy saved;
1277 bool using_guc = intel_engine_uses_guc(engine);
1278 bool ok;
1279 int ret2;
1280
1281 pr_info("Verifying after %s reset...\n", engine->name);
1282 ret = intel_selftest_modify_policy(engine, &saved,
1283 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1284 if (ret)
1285 break;
1286
1287 ce = intel_context_create(engine);
1288 if (IS_ERR(ce)) {
1289 ret = PTR_ERR(ce);
1290 goto restore;
1291 }
1292
1293 if (!using_guc) {
1294 ok = verify_wa_lists(gt, lists, "before reset");
1295 if (!ok) {
1296 ret = -ESRCH;
1297 goto err;
1298 }
1299
1300 ret = intel_engine_reset(engine, "live_workarounds:idle");
1301 if (ret) {
1302 pr_err("%s: Reset failed while idle\n", engine->name);
1303 goto err;
1304 }
1305
1306 ok = verify_wa_lists(gt, lists, "after idle reset");
1307 if (!ok) {
1308 ret = -ESRCH;
1309 goto err;
1310 }
1311 }
1312
1313 ret = igt_spinner_init(&spin, engine->gt);
1314 if (ret)
1315 goto err;
1316
1317 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1318 if (IS_ERR(rq)) {
1319 ret = PTR_ERR(rq);
1320 igt_spinner_fini(&spin);
1321 goto err;
1322 }
1323
1324 ret = request_add_spin(rq, &spin);
1325 if (ret) {
1326 pr_err("%s: Spinner failed to start\n", engine->name);
1327 igt_spinner_fini(&spin);
1328 goto err;
1329 }
1330
1331
1332 if (i915_request_completed(rq)) {
1333 ret = -ETIMEDOUT;
1334 goto skip;
1335 }
1336
1337 if (!using_guc) {
1338 ret = intel_engine_reset(engine, "live_workarounds:active");
1339 if (ret) {
1340 pr_err("%s: Reset failed on an active spinner\n",
1341 engine->name);
1342 igt_spinner_fini(&spin);
1343 goto err;
1344 }
1345 }
1346
1347
1348 if (ret == 0)
1349 ret = intel_selftest_wait_for_rq(rq);
1350
1351skip:
1352 igt_spinner_end(&spin);
1353 igt_spinner_fini(&spin);
1354
1355 ok = verify_wa_lists(gt, lists, "after busy reset");
1356 if (!ok)
1357 ret = -ESRCH;
1358
1359err:
1360 intel_context_put(ce);
1361
1362restore:
1363 ret2 = intel_selftest_restore_policy(engine, &saved);
1364 if (ret == 0)
1365 ret = ret2;
1366 if (ret)
1367 break;
1368 }
1369
1370 reference_lists_fini(gt, lists);
1371 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1372 igt_global_reset_unlock(gt);
1373 kfree(lists);
1374
1375 igt_flush_test(gt->i915);
1376
1377 return ret;
1378}
1379
1380int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1381{
1382 static const struct i915_subtest tests[] = {
1383 SUBTEST(live_dirty_whitelist),
1384 SUBTEST(live_reset_whitelist),
1385 SUBTEST(live_isolated_whitelist),
1386 SUBTEST(live_gpu_reset_workarounds),
1387 SUBTEST(live_engine_reset_workarounds),
1388 };
1389
1390 if (intel_gt_is_wedged(&i915->gt))
1391 return 0;
1392
1393 return intel_gt_live_subtests(tests, &i915->gt);
1394}
1395
