linux/drivers/gpu/drm/i915/gt/selftest_workarounds.c
<<
>>
Prefs
   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2018 Intel Corporation
   4 */
   5
   6#include "gem/i915_gem_pm.h"
   7#include "gt/intel_engine_user.h"
   8#include "gt/intel_gt.h"
   9#include "i915_selftest.h"
  10#include "intel_reset.h"
  11
  12#include "selftests/igt_flush_test.h"
  13#include "selftests/igt_reset.h"
  14#include "selftests/igt_spinner.h"
  15#include "selftests/intel_scheduler_helpers.h"
  16#include "selftests/mock_drm.h"
  17
  18#include "gem/selftests/igt_gem_utils.h"
  19#include "gem/selftests/mock_context.h"
  20
  21static const struct wo_register {
  22        enum intel_platform platform;
  23        u32 reg;
  24} wo_registers[] = {
  25        { INTEL_GEMINILAKE, 0x731c }
  26};
  27
  28struct wa_lists {
  29        struct i915_wa_list gt_wa_list;
  30        struct {
  31                struct i915_wa_list wa_list;
  32                struct i915_wa_list ctx_wa_list;
  33        } engine[I915_NUM_ENGINES];
  34};
  35
  36static int request_add_sync(struct i915_request *rq, int err)
  37{
  38        i915_request_get(rq);
  39        i915_request_add(rq);
  40        if (i915_request_wait(rq, 0, HZ / 5) < 0)
  41                err = -EIO;
  42        i915_request_put(rq);
  43
  44        return err;
  45}
  46
  47static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
  48{
  49        int err = 0;
  50
  51        i915_request_get(rq);
  52        i915_request_add(rq);
  53        if (spin && !igt_wait_for_spinner(spin, rq))
  54                err = -ETIMEDOUT;
  55        i915_request_put(rq);
  56
  57        return err;
  58}
  59
  60static void
  61reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
  62{
  63        struct intel_engine_cs *engine;
  64        enum intel_engine_id id;
  65
  66        memset(lists, 0, sizeof(*lists));
  67
  68        wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
  69        gt_init_workarounds(gt, &lists->gt_wa_list);
  70        wa_init_finish(&lists->gt_wa_list);
  71
  72        for_each_engine(engine, gt, id) {
  73                struct i915_wa_list *wal = &lists->engine[id].wa_list;
  74
  75                wa_init_start(wal, "REF", engine->name);
  76                engine_init_workarounds(engine, wal);
  77                wa_init_finish(wal);
  78
  79                __intel_engine_init_ctx_wa(engine,
  80                                           &lists->engine[id].ctx_wa_list,
  81                                           "CTX_REF");
  82        }
  83}
  84
  85static void
  86reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
  87{
  88        struct intel_engine_cs *engine;
  89        enum intel_engine_id id;
  90
  91        for_each_engine(engine, gt, id)
  92                intel_wa_list_free(&lists->engine[id].wa_list);
  93
  94        intel_wa_list_free(&lists->gt_wa_list);
  95}
  96
  97static struct drm_i915_gem_object *
  98read_nonprivs(struct intel_context *ce)
  99{
 100        struct intel_engine_cs *engine = ce->engine;
 101        const u32 base = engine->mmio_base;
 102        struct drm_i915_gem_object *result;
 103        struct i915_request *rq;
 104        struct i915_vma *vma;
 105        u32 srm, *cs;
 106        int err;
 107        int i;
 108
 109        result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
 110        if (IS_ERR(result))
 111                return result;
 112
 113        i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
 114
 115        cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
 116        if (IS_ERR(cs)) {
 117                err = PTR_ERR(cs);
 118                goto err_obj;
 119        }
 120        memset(cs, 0xc5, PAGE_SIZE);
 121        i915_gem_object_flush_map(result);
 122        i915_gem_object_unpin_map(result);
 123
 124        vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
 125        if (IS_ERR(vma)) {
 126                err = PTR_ERR(vma);
 127                goto err_obj;
 128        }
 129
 130        err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
 131        if (err)
 132                goto err_obj;
 133
 134        rq = intel_context_create_request(ce);
 135        if (IS_ERR(rq)) {
 136                err = PTR_ERR(rq);
 137                goto err_pin;
 138        }
 139
 140        i915_vma_lock(vma);
 141        err = i915_request_await_object(rq, vma->obj, true);
 142        if (err == 0)
 143                err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 144        i915_vma_unlock(vma);
 145        if (err)
 146                goto err_req;
 147
 148        srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 149        if (GRAPHICS_VER(engine->i915) >= 8)
 150                srm++;
 151
 152        cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
 153        if (IS_ERR(cs)) {
 154                err = PTR_ERR(cs);
 155                goto err_req;
 156        }
 157
 158        for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 159                *cs++ = srm;
 160                *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
 161                *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
 162                *cs++ = 0;
 163        }
 164        intel_ring_advance(rq, cs);
 165
 166        i915_request_add(rq);
 167        i915_vma_unpin(vma);
 168
 169        return result;
 170
 171err_req:
 172        i915_request_add(rq);
 173err_pin:
 174        i915_vma_unpin(vma);
 175err_obj:
 176        i915_gem_object_put(result);
 177        return ERR_PTR(err);
 178}
 179
 180static u32
 181get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
 182{
 183        i915_reg_t reg = i < engine->whitelist.count ?
 184                         engine->whitelist.list[i].reg :
 185                         RING_NOPID(engine->mmio_base);
 186
 187        return i915_mmio_reg_offset(reg);
 188}
 189
 190static void
 191print_results(const struct intel_engine_cs *engine, const u32 *results)
 192{
 193        unsigned int i;
 194
 195        for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 196                u32 expected = get_whitelist_reg(engine, i);
 197                u32 actual = results[i];
 198
 199                pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
 200                        i, expected, actual);
 201        }
 202}
 203
 204static int check_whitelist(struct intel_context *ce)
 205{
 206        struct intel_engine_cs *engine = ce->engine;
 207        struct drm_i915_gem_object *results;
 208        struct intel_wedge_me wedge;
 209        u32 *vaddr;
 210        int err;
 211        int i;
 212
 213        results = read_nonprivs(ce);
 214        if (IS_ERR(results))
 215                return PTR_ERR(results);
 216
 217        err = 0;
 218        i915_gem_object_lock(results, NULL);
 219        intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
 220                err = i915_gem_object_set_to_cpu_domain(results, false);
 221
 222        if (intel_gt_is_wedged(engine->gt))
 223                err = -EIO;
 224        if (err)
 225                goto out_put;
 226
 227        vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
 228        if (IS_ERR(vaddr)) {
 229                err = PTR_ERR(vaddr);
 230                goto out_put;
 231        }
 232
 233        for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
 234                u32 expected = get_whitelist_reg(engine, i);
 235                u32 actual = vaddr[i];
 236
 237                if (expected != actual) {
 238                        print_results(engine, vaddr);
 239                        pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
 240                               i, expected, actual);
 241
 242                        err = -EINVAL;
 243                        break;
 244                }
 245        }
 246
 247        i915_gem_object_unpin_map(results);
 248out_put:
 249        i915_gem_object_unlock(results);
 250        i915_gem_object_put(results);
 251        return err;
 252}
 253
 254static int do_device_reset(struct intel_engine_cs *engine)
 255{
 256        intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
 257        return 0;
 258}
 259
 260static int do_engine_reset(struct intel_engine_cs *engine)
 261{
 262        return intel_engine_reset(engine, "live_workarounds");
 263}
 264
 265static int do_guc_reset(struct intel_engine_cs *engine)
 266{
 267        /* Currently a no-op as the reset is handled by GuC */
 268        return 0;
 269}
 270
 271static int
 272switch_to_scratch_context(struct intel_engine_cs *engine,
 273                          struct igt_spinner *spin,
 274                          struct i915_request **rq)
 275{
 276        struct intel_context *ce;
 277        int err = 0;
 278
 279        ce = intel_context_create(engine);
 280        if (IS_ERR(ce))
 281                return PTR_ERR(ce);
 282
 283        *rq = igt_spinner_create_request(spin, ce, MI_NOOP);
 284        intel_context_put(ce);
 285
 286        if (IS_ERR(*rq)) {
 287                spin = NULL;
 288                err = PTR_ERR(*rq);
 289                goto err;
 290        }
 291
 292        err = request_add_spin(*rq, spin);
 293err:
 294        if (err && spin)
 295                igt_spinner_end(spin);
 296
 297        return err;
 298}
 299
 300static int check_whitelist_across_reset(struct intel_engine_cs *engine,
 301                                        int (*reset)(struct intel_engine_cs *),
 302                                        const char *name)
 303{
 304        struct intel_context *ce, *tmp;
 305        struct igt_spinner spin;
 306        struct i915_request *rq;
 307        intel_wakeref_t wakeref;
 308        int err;
 309
 310        pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
 311                engine->whitelist.count, engine->name, name);
 312
 313        ce = intel_context_create(engine);
 314        if (IS_ERR(ce))
 315                return PTR_ERR(ce);
 316
 317        err = igt_spinner_init(&spin, engine->gt);
 318        if (err)
 319                goto out_ctx;
 320
 321        err = check_whitelist(ce);
 322        if (err) {
 323                pr_err("Invalid whitelist *before* %s reset!\n", name);
 324                goto out_spin;
 325        }
 326
 327        err = switch_to_scratch_context(engine, &spin, &rq);
 328        if (err)
 329                goto out_spin;
 330
 331        /* Ensure the spinner hasn't aborted */
 332        if (i915_request_completed(rq)) {
 333                pr_err("%s spinner failed to start\n", name);
 334                err = -ETIMEDOUT;
 335                goto out_spin;
 336        }
 337
 338        with_intel_runtime_pm(engine->uncore->rpm, wakeref)
 339                err = reset(engine);
 340
 341        /* Ensure the reset happens and kills the engine */
 342        if (err == 0)
 343                err = intel_selftest_wait_for_rq(rq);
 344
 345        igt_spinner_end(&spin);
 346
 347        if (err) {
 348                pr_err("%s reset failed\n", name);
 349                goto out_spin;
 350        }
 351
 352        err = check_whitelist(ce);
 353        if (err) {
 354                pr_err("Whitelist not preserved in context across %s reset!\n",
 355                       name);
 356                goto out_spin;
 357        }
 358
 359        tmp = intel_context_create(engine);
 360        if (IS_ERR(tmp)) {
 361                err = PTR_ERR(tmp);
 362                goto out_spin;
 363        }
 364        intel_context_put(ce);
 365        ce = tmp;
 366
 367        err = check_whitelist(ce);
 368        if (err) {
 369                pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
 370                       name);
 371                goto out_spin;
 372        }
 373
 374out_spin:
 375        igt_spinner_fini(&spin);
 376out_ctx:
 377        intel_context_put(ce);
 378        return err;
 379}
 380
 381static struct i915_vma *create_batch(struct i915_address_space *vm)
 382{
 383        struct drm_i915_gem_object *obj;
 384        struct i915_vma *vma;
 385        int err;
 386
 387        obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
 388        if (IS_ERR(obj))
 389                return ERR_CAST(obj);
 390
 391        vma = i915_vma_instance(obj, vm, NULL);
 392        if (IS_ERR(vma)) {
 393                err = PTR_ERR(vma);
 394                goto err_obj;
 395        }
 396
 397        err = i915_vma_pin(vma, 0, 0, PIN_USER);
 398        if (err)
 399                goto err_obj;
 400
 401        return vma;
 402
 403err_obj:
 404        i915_gem_object_put(obj);
 405        return ERR_PTR(err);
 406}
 407
 408static u32 reg_write(u32 old, u32 new, u32 rsvd)
 409{
 410        if (rsvd == 0x0000ffff) {
 411                old &= ~(new >> 16);
 412                old |= new & (new >> 16);
 413        } else {
 414                old &= ~rsvd;
 415                old |= new & rsvd;
 416        }
 417
 418        return old;
 419}
 420
 421static bool wo_register(struct intel_engine_cs *engine, u32 reg)
 422{
 423        enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
 424        int i;
 425
 426        if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 427             RING_FORCE_TO_NONPRIV_ACCESS_WR)
 428                return true;
 429
 430        for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
 431                if (wo_registers[i].platform == platform &&
 432                    wo_registers[i].reg == reg)
 433                        return true;
 434        }
 435
 436        return false;
 437}
 438
 439static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
 440{
 441        reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
 442        switch (reg) {
 443        case 0x358:
 444        case 0x35c:
 445        case 0x3a8:
 446                return true;
 447
 448        default:
 449                return false;
 450        }
 451}
 452
 453static bool ro_register(u32 reg)
 454{
 455        if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
 456             RING_FORCE_TO_NONPRIV_ACCESS_RD)
 457                return true;
 458
 459        return false;
 460}
 461
 462static int whitelist_writable_count(struct intel_engine_cs *engine)
 463{
 464        int count = engine->whitelist.count;
 465        int i;
 466
 467        for (i = 0; i < engine->whitelist.count; i++) {
 468                u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 469
 470                if (ro_register(reg))
 471                        count--;
 472        }
 473
 474        return count;
 475}
 476
 477static int check_dirty_whitelist(struct intel_context *ce)
 478{
 479        const u32 values[] = {
 480                0x00000000,
 481                0x01010101,
 482                0x10100101,
 483                0x03030303,
 484                0x30300303,
 485                0x05050505,
 486                0x50500505,
 487                0x0f0f0f0f,
 488                0xf00ff00f,
 489                0x10101010,
 490                0xf0f01010,
 491                0x30303030,
 492                0xa0a03030,
 493                0x50505050,
 494                0xc0c05050,
 495                0xf0f0f0f0,
 496                0x11111111,
 497                0x33333333,
 498                0x55555555,
 499                0x0000ffff,
 500                0x00ff00ff,
 501                0xff0000ff,
 502                0xffff00ff,
 503                0xffffffff,
 504        };
 505        struct intel_engine_cs *engine = ce->engine;
 506        struct i915_vma *scratch;
 507        struct i915_vma *batch;
 508        int err = 0, i, v, sz;
 509        u32 *cs, *results;
 510
 511        sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
 512        scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
 513        if (IS_ERR(scratch))
 514                return PTR_ERR(scratch);
 515
 516        batch = create_batch(ce->vm);
 517        if (IS_ERR(batch)) {
 518                err = PTR_ERR(batch);
 519                goto out_scratch;
 520        }
 521
 522        for (i = 0; i < engine->whitelist.count; i++) {
 523                u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 524                struct i915_gem_ww_ctx ww;
 525                u64 addr = scratch->node.start;
 526                struct i915_request *rq;
 527                u32 srm, lrm, rsvd;
 528                u32 expect;
 529                int idx;
 530                bool ro_reg;
 531
 532                if (wo_register(engine, reg))
 533                        continue;
 534
 535                if (timestamp(engine, reg))
 536                        continue; /* timestamps are expected to autoincrement */
 537
 538                ro_reg = ro_register(reg);
 539
 540                i915_gem_ww_ctx_init(&ww, false);
 541retry:
 542                cs = NULL;
 543                err = i915_gem_object_lock(scratch->obj, &ww);
 544                if (!err)
 545                        err = i915_gem_object_lock(batch->obj, &ww);
 546                if (!err)
 547                        err = intel_context_pin_ww(ce, &ww);
 548                if (err)
 549                        goto out;
 550
 551                cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
 552                if (IS_ERR(cs)) {
 553                        err = PTR_ERR(cs);
 554                        goto out_ctx;
 555                }
 556
 557                results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
 558                if (IS_ERR(results)) {
 559                        err = PTR_ERR(results);
 560                        goto out_unmap_batch;
 561                }
 562
 563                /* Clear non priv flags */
 564                reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 565
 566                srm = MI_STORE_REGISTER_MEM;
 567                lrm = MI_LOAD_REGISTER_MEM;
 568                if (GRAPHICS_VER(engine->i915) >= 8)
 569                        lrm++, srm++;
 570
 571                pr_debug("%s: Writing garbage to %x\n",
 572                         engine->name, reg);
 573
 574                /* SRM original */
 575                *cs++ = srm;
 576                *cs++ = reg;
 577                *cs++ = lower_32_bits(addr);
 578                *cs++ = upper_32_bits(addr);
 579
 580                idx = 1;
 581                for (v = 0; v < ARRAY_SIZE(values); v++) {
 582                        /* LRI garbage */
 583                        *cs++ = MI_LOAD_REGISTER_IMM(1);
 584                        *cs++ = reg;
 585                        *cs++ = values[v];
 586
 587                        /* SRM result */
 588                        *cs++ = srm;
 589                        *cs++ = reg;
 590                        *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 591                        *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 592                        idx++;
 593                }
 594                for (v = 0; v < ARRAY_SIZE(values); v++) {
 595                        /* LRI garbage */
 596                        *cs++ = MI_LOAD_REGISTER_IMM(1);
 597                        *cs++ = reg;
 598                        *cs++ = ~values[v];
 599
 600                        /* SRM result */
 601                        *cs++ = srm;
 602                        *cs++ = reg;
 603                        *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
 604                        *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
 605                        idx++;
 606                }
 607                GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
 608
 609                /* LRM original -- don't leave garbage in the context! */
 610                *cs++ = lrm;
 611                *cs++ = reg;
 612                *cs++ = lower_32_bits(addr);
 613                *cs++ = upper_32_bits(addr);
 614
 615                *cs++ = MI_BATCH_BUFFER_END;
 616
 617                i915_gem_object_flush_map(batch->obj);
 618                i915_gem_object_unpin_map(batch->obj);
 619                intel_gt_chipset_flush(engine->gt);
 620                cs = NULL;
 621
 622                rq = i915_request_create(ce);
 623                if (IS_ERR(rq)) {
 624                        err = PTR_ERR(rq);
 625                        goto out_unmap_scratch;
 626                }
 627
 628                if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 629                        err = engine->emit_init_breadcrumb(rq);
 630                        if (err)
 631                                goto err_request;
 632                }
 633
 634                err = i915_request_await_object(rq, batch->obj, false);
 635                if (err == 0)
 636                        err = i915_vma_move_to_active(batch, rq, 0);
 637                if (err)
 638                        goto err_request;
 639
 640                err = i915_request_await_object(rq, scratch->obj, true);
 641                if (err == 0)
 642                        err = i915_vma_move_to_active(scratch, rq,
 643                                                      EXEC_OBJECT_WRITE);
 644                if (err)
 645                        goto err_request;
 646
 647                err = engine->emit_bb_start(rq,
 648                                            batch->node.start, PAGE_SIZE,
 649                                            0);
 650                if (err)
 651                        goto err_request;
 652
 653err_request:
 654                err = request_add_sync(rq, err);
 655                if (err) {
 656                        pr_err("%s: Futzing %x timedout; cancelling test\n",
 657                               engine->name, reg);
 658                        intel_gt_set_wedged(engine->gt);
 659                        goto out_unmap_scratch;
 660                }
 661
 662                GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
 663                if (!ro_reg) {
 664                        /* detect write masking */
 665                        rsvd = results[ARRAY_SIZE(values)];
 666                        if (!rsvd) {
 667                                pr_err("%s: Unable to write to whitelisted register %x\n",
 668                                       engine->name, reg);
 669                                err = -EINVAL;
 670                                goto out_unmap_scratch;
 671                        }
 672                } else {
 673                        rsvd = 0;
 674                }
 675
 676                expect = results[0];
 677                idx = 1;
 678                for (v = 0; v < ARRAY_SIZE(values); v++) {
 679                        if (ro_reg)
 680                                expect = results[0];
 681                        else
 682                                expect = reg_write(expect, values[v], rsvd);
 683
 684                        if (results[idx] != expect)
 685                                err++;
 686                        idx++;
 687                }
 688                for (v = 0; v < ARRAY_SIZE(values); v++) {
 689                        if (ro_reg)
 690                                expect = results[0];
 691                        else
 692                                expect = reg_write(expect, ~values[v], rsvd);
 693
 694                        if (results[idx] != expect)
 695                                err++;
 696                        idx++;
 697                }
 698                if (err) {
 699                        pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
 700                               engine->name, err, reg);
 701
 702                        if (ro_reg)
 703                                pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
 704                                        engine->name, reg, results[0]);
 705                        else
 706                                pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
 707                                        engine->name, reg, results[0], rsvd);
 708
 709                        expect = results[0];
 710                        idx = 1;
 711                        for (v = 0; v < ARRAY_SIZE(values); v++) {
 712                                u32 w = values[v];
 713
 714                                if (ro_reg)
 715                                        expect = results[0];
 716                                else
 717                                        expect = reg_write(expect, w, rsvd);
 718                                pr_info("Wrote %08x, read %08x, expect %08x\n",
 719                                        w, results[idx], expect);
 720                                idx++;
 721                        }
 722                        for (v = 0; v < ARRAY_SIZE(values); v++) {
 723                                u32 w = ~values[v];
 724
 725                                if (ro_reg)
 726                                        expect = results[0];
 727                                else
 728                                        expect = reg_write(expect, w, rsvd);
 729                                pr_info("Wrote %08x, read %08x, expect %08x\n",
 730                                        w, results[idx], expect);
 731                                idx++;
 732                        }
 733
 734                        err = -EINVAL;
 735                }
 736out_unmap_scratch:
 737                i915_gem_object_unpin_map(scratch->obj);
 738out_unmap_batch:
 739                if (cs)
 740                        i915_gem_object_unpin_map(batch->obj);
 741out_ctx:
 742                intel_context_unpin(ce);
 743out:
 744                if (err == -EDEADLK) {
 745                        err = i915_gem_ww_ctx_backoff(&ww);
 746                        if (!err)
 747                                goto retry;
 748                }
 749                i915_gem_ww_ctx_fini(&ww);
 750                if (err)
 751                        break;
 752        }
 753
 754        if (igt_flush_test(engine->i915))
 755                err = -EIO;
 756
 757        i915_vma_unpin_and_release(&batch, 0);
 758out_scratch:
 759        i915_vma_unpin_and_release(&scratch, 0);
 760        return err;
 761}
 762
 763static int live_dirty_whitelist(void *arg)
 764{
 765        struct intel_gt *gt = arg;
 766        struct intel_engine_cs *engine;
 767        enum intel_engine_id id;
 768
 769        /* Can the user write to the whitelisted registers? */
 770
 771        if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
 772                return 0;
 773
 774        for_each_engine(engine, gt, id) {
 775                struct intel_context *ce;
 776                int err;
 777
 778                if (engine->whitelist.count == 0)
 779                        continue;
 780
 781                ce = intel_context_create(engine);
 782                if (IS_ERR(ce))
 783                        return PTR_ERR(ce);
 784
 785                err = check_dirty_whitelist(ce);
 786                intel_context_put(ce);
 787                if (err)
 788                        return err;
 789        }
 790
 791        return 0;
 792}
 793
 794static int live_reset_whitelist(void *arg)
 795{
 796        struct intel_gt *gt = arg;
 797        struct intel_engine_cs *engine;
 798        enum intel_engine_id id;
 799        int err = 0;
 800
 801        /* If we reset the gpu, we should not lose the RING_NONPRIV */
 802        igt_global_reset_lock(gt);
 803
 804        for_each_engine(engine, gt, id) {
 805                if (engine->whitelist.count == 0)
 806                        continue;
 807
 808                if (intel_has_reset_engine(gt)) {
 809                        if (intel_engine_uses_guc(engine)) {
 810                                struct intel_selftest_saved_policy saved;
 811                                int err2;
 812
 813                                err = intel_selftest_modify_policy(engine, &saved,
 814                                                                   SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
 815                                if (err)
 816                                        goto out;
 817
 818                                err = check_whitelist_across_reset(engine,
 819                                                                   do_guc_reset,
 820                                                                   "guc");
 821
 822                                err2 = intel_selftest_restore_policy(engine, &saved);
 823                                if (err == 0)
 824                                        err = err2;
 825                        } else {
 826                                err = check_whitelist_across_reset(engine,
 827                                                                   do_engine_reset,
 828                                                                   "engine");
 829                        }
 830
 831                        if (err)
 832                                goto out;
 833                }
 834
 835                if (intel_has_gpu_reset(gt)) {
 836                        err = check_whitelist_across_reset(engine,
 837                                                           do_device_reset,
 838                                                           "device");
 839                        if (err)
 840                                goto out;
 841                }
 842        }
 843
 844out:
 845        igt_global_reset_unlock(gt);
 846        return err;
 847}
 848
 849static int read_whitelisted_registers(struct intel_context *ce,
 850                                      struct i915_vma *results)
 851{
 852        struct intel_engine_cs *engine = ce->engine;
 853        struct i915_request *rq;
 854        int i, err = 0;
 855        u32 srm, *cs;
 856
 857        rq = intel_context_create_request(ce);
 858        if (IS_ERR(rq))
 859                return PTR_ERR(rq);
 860
 861        i915_vma_lock(results);
 862        err = i915_request_await_object(rq, results->obj, true);
 863        if (err == 0)
 864                err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
 865        i915_vma_unlock(results);
 866        if (err)
 867                goto err_req;
 868
 869        srm = MI_STORE_REGISTER_MEM;
 870        if (GRAPHICS_VER(engine->i915) >= 8)
 871                srm++;
 872
 873        cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
 874        if (IS_ERR(cs)) {
 875                err = PTR_ERR(cs);
 876                goto err_req;
 877        }
 878
 879        for (i = 0; i < engine->whitelist.count; i++) {
 880                u64 offset = results->node.start + sizeof(u32) * i;
 881                u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 882
 883                /* Clear non priv flags */
 884                reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 885
 886                *cs++ = srm;
 887                *cs++ = reg;
 888                *cs++ = lower_32_bits(offset);
 889                *cs++ = upper_32_bits(offset);
 890        }
 891        intel_ring_advance(rq, cs);
 892
 893err_req:
 894        return request_add_sync(rq, err);
 895}
 896
 897static int scrub_whitelisted_registers(struct intel_context *ce)
 898{
 899        struct intel_engine_cs *engine = ce->engine;
 900        struct i915_request *rq;
 901        struct i915_vma *batch;
 902        int i, err = 0;
 903        u32 *cs;
 904
 905        batch = create_batch(ce->vm);
 906        if (IS_ERR(batch))
 907                return PTR_ERR(batch);
 908
 909        cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
 910        if (IS_ERR(cs)) {
 911                err = PTR_ERR(cs);
 912                goto err_batch;
 913        }
 914
 915        *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
 916        for (i = 0; i < engine->whitelist.count; i++) {
 917                u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
 918
 919                if (ro_register(reg))
 920                        continue;
 921
 922                /* Clear non priv flags */
 923                reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
 924
 925                *cs++ = reg;
 926                *cs++ = 0xffffffff;
 927        }
 928        *cs++ = MI_BATCH_BUFFER_END;
 929
 930        i915_gem_object_flush_map(batch->obj);
 931        intel_gt_chipset_flush(engine->gt);
 932
 933        rq = intel_context_create_request(ce);
 934        if (IS_ERR(rq)) {
 935                err = PTR_ERR(rq);
 936                goto err_unpin;
 937        }
 938
 939        if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
 940                err = engine->emit_init_breadcrumb(rq);
 941                if (err)
 942                        goto err_request;
 943        }
 944
 945        i915_vma_lock(batch);
 946        err = i915_request_await_object(rq, batch->obj, false);
 947        if (err == 0)
 948                err = i915_vma_move_to_active(batch, rq, 0);
 949        i915_vma_unlock(batch);
 950        if (err)
 951                goto err_request;
 952
 953        /* Perform the writes from an unprivileged "user" batch */
 954        err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
 955
 956err_request:
 957        err = request_add_sync(rq, err);
 958
 959err_unpin:
 960        i915_gem_object_unpin_map(batch->obj);
 961err_batch:
 962        i915_vma_unpin_and_release(&batch, 0);
 963        return err;
 964}
 965
 966struct regmask {
 967        i915_reg_t reg;
 968        u8 graphics_ver;
 969};
 970
 971static bool find_reg(struct drm_i915_private *i915,
 972                     i915_reg_t reg,
 973                     const struct regmask *tbl,
 974                     unsigned long count)
 975{
 976        u32 offset = i915_mmio_reg_offset(reg);
 977
 978        while (count--) {
 979                if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
 980                    i915_mmio_reg_offset(tbl->reg) == offset)
 981                        return true;
 982                tbl++;
 983        }
 984
 985        return false;
 986}
 987
 988static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
 989{
 990        /* Alas, we must pardon some whitelists. Mistakes already made */
 991        static const struct regmask pardon[] = {
 992                { GEN9_CTX_PREEMPT_REG, 9 },
 993                { GEN8_L3SQCREG4, 9 },
 994        };
 995
 996        return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
 997}
 998
 999static bool result_eq(struct intel_engine_cs *engine,
1000                      u32 a, u32 b, i915_reg_t reg)
1001{
1002        if (a != b && !pardon_reg(engine->i915, reg)) {
1003                pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
1004                       i915_mmio_reg_offset(reg), a, b);
1005                return false;
1006        }
1007
1008        return true;
1009}
1010
1011static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
1012{
1013        /* Some registers do not seem to behave and our writes unreadable */
1014        static const struct regmask wo[] = {
1015                { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
1016        };
1017
1018        return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
1019}
1020
1021static bool result_neq(struct intel_engine_cs *engine,
1022                       u32 a, u32 b, i915_reg_t reg)
1023{
1024        if (a == b && !writeonly_reg(engine->i915, reg)) {
1025                pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
1026                       i915_mmio_reg_offset(reg), a);
1027                return false;
1028        }
1029
1030        return true;
1031}
1032
1033static int
1034check_whitelisted_registers(struct intel_engine_cs *engine,
1035                            struct i915_vma *A,
1036                            struct i915_vma *B,
1037                            bool (*fn)(struct intel_engine_cs *engine,
1038                                       u32 a, u32 b,
1039                                       i915_reg_t reg))
1040{
1041        u32 *a, *b;
1042        int i, err;
1043
1044        a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1045        if (IS_ERR(a))
1046                return PTR_ERR(a);
1047
1048        b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1049        if (IS_ERR(b)) {
1050                err = PTR_ERR(b);
1051                goto err_a;
1052        }
1053
1054        err = 0;
1055        for (i = 0; i < engine->whitelist.count; i++) {
1056                const struct i915_wa *wa = &engine->whitelist.list[i];
1057
1058                if (i915_mmio_reg_offset(wa->reg) &
1059                    RING_FORCE_TO_NONPRIV_ACCESS_RD)
1060                        continue;
1061
1062                if (!fn(engine, a[i], b[i], wa->reg))
1063                        err = -EINVAL;
1064        }
1065
1066        i915_gem_object_unpin_map(B->obj);
1067err_a:
1068        i915_gem_object_unpin_map(A->obj);
1069        return err;
1070}
1071
1072static int live_isolated_whitelist(void *arg)
1073{
1074        struct intel_gt *gt = arg;
1075        struct {
1076                struct i915_vma *scratch[2];
1077        } client[2] = {};
1078        struct intel_engine_cs *engine;
1079        enum intel_engine_id id;
1080        int i, err = 0;
1081
1082        /*
1083         * Check that a write into a whitelist register works, but
1084         * invisible to a second context.
1085         */
1086
1087        if (!intel_engines_has_context_isolation(gt->i915))
1088                return 0;
1089
1090        for (i = 0; i < ARRAY_SIZE(client); i++) {
1091                client[i].scratch[0] =
1092                        __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1093                if (IS_ERR(client[i].scratch[0])) {
1094                        err = PTR_ERR(client[i].scratch[0]);
1095                        goto err;
1096                }
1097
1098                client[i].scratch[1] =
1099                        __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1100                if (IS_ERR(client[i].scratch[1])) {
1101                        err = PTR_ERR(client[i].scratch[1]);
1102                        i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1103                        goto err;
1104                }
1105        }
1106
1107        for_each_engine(engine, gt, id) {
1108                struct intel_context *ce[2];
1109
1110                if (!engine->kernel_context->vm)
1111                        continue;
1112
1113                if (!whitelist_writable_count(engine))
1114                        continue;
1115
1116                ce[0] = intel_context_create(engine);
1117                if (IS_ERR(ce[0])) {
1118                        err = PTR_ERR(ce[0]);
1119                        break;
1120                }
1121                ce[1] = intel_context_create(engine);
1122                if (IS_ERR(ce[1])) {
1123                        err = PTR_ERR(ce[1]);
1124                        intel_context_put(ce[0]);
1125                        break;
1126                }
1127
1128                /* Read default values */
1129                err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1130                if (err)
1131                        goto err_ce;
1132
1133                /* Try to overwrite registers (should only affect ctx0) */
1134                err = scrub_whitelisted_registers(ce[0]);
1135                if (err)
1136                        goto err_ce;
1137
1138                /* Read values from ctx1, we expect these to be defaults */
1139                err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1140                if (err)
1141                        goto err_ce;
1142
1143                /* Verify that both reads return the same default values */
1144                err = check_whitelisted_registers(engine,
1145                                                  client[0].scratch[0],
1146                                                  client[1].scratch[0],
1147                                                  result_eq);
1148                if (err)
1149                        goto err_ce;
1150
1151                /* Read back the updated values in ctx0 */
1152                err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1153                if (err)
1154                        goto err_ce;
1155
1156                /* User should be granted privilege to overwhite regs */
1157                err = check_whitelisted_registers(engine,
1158                                                  client[0].scratch[0],
1159                                                  client[0].scratch[1],
1160                                                  result_neq);
1161err_ce:
1162                intel_context_put(ce[1]);
1163                intel_context_put(ce[0]);
1164                if (err)
1165                        break;
1166        }
1167
1168err:
1169        for (i = 0; i < ARRAY_SIZE(client); i++) {
1170                i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1171                i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1172        }
1173
1174        if (igt_flush_test(gt->i915))
1175                err = -EIO;
1176
1177        return err;
1178}
1179
1180static bool
1181verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1182                const char *str)
1183{
1184        struct intel_engine_cs *engine;
1185        enum intel_engine_id id;
1186        bool ok = true;
1187
1188        ok &= wa_list_verify(gt, &lists->gt_wa_list, str);
1189
1190        for_each_engine(engine, gt, id) {
1191                struct intel_context *ce;
1192
1193                ce = intel_context_create(engine);
1194                if (IS_ERR(ce))
1195                        return false;
1196
1197                ok &= engine_wa_list_verify(ce,
1198                                            &lists->engine[id].wa_list,
1199                                            str) == 0;
1200
1201                ok &= engine_wa_list_verify(ce,
1202                                            &lists->engine[id].ctx_wa_list,
1203                                            str) == 0;
1204
1205                intel_context_put(ce);
1206        }
1207
1208        return ok;
1209}
1210
1211static int
1212live_gpu_reset_workarounds(void *arg)
1213{
1214        struct intel_gt *gt = arg;
1215        intel_wakeref_t wakeref;
1216        struct wa_lists *lists;
1217        bool ok;
1218
1219        if (!intel_has_gpu_reset(gt))
1220                return 0;
1221
1222        lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1223        if (!lists)
1224                return -ENOMEM;
1225
1226        pr_info("Verifying after GPU reset...\n");
1227
1228        igt_global_reset_lock(gt);
1229        wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1230
1231        reference_lists_init(gt, lists);
1232
1233        ok = verify_wa_lists(gt, lists, "before reset");
1234        if (!ok)
1235                goto out;
1236
1237        intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1238
1239        ok = verify_wa_lists(gt, lists, "after reset");
1240
1241out:
1242        reference_lists_fini(gt, lists);
1243        intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1244        igt_global_reset_unlock(gt);
1245        kfree(lists);
1246
1247        return ok ? 0 : -ESRCH;
1248}
1249
1250static int
1251live_engine_reset_workarounds(void *arg)
1252{
1253        struct intel_gt *gt = arg;
1254        struct intel_engine_cs *engine;
1255        enum intel_engine_id id;
1256        struct intel_context *ce;
1257        struct igt_spinner spin;
1258        struct i915_request *rq;
1259        intel_wakeref_t wakeref;
1260        struct wa_lists *lists;
1261        int ret = 0;
1262
1263        if (!intel_has_reset_engine(gt))
1264                return 0;
1265
1266        lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1267        if (!lists)
1268                return -ENOMEM;
1269
1270        igt_global_reset_lock(gt);
1271        wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1272
1273        reference_lists_init(gt, lists);
1274
1275        for_each_engine(engine, gt, id) {
1276                struct intel_selftest_saved_policy saved;
1277                bool using_guc = intel_engine_uses_guc(engine);
1278                bool ok;
1279                int ret2;
1280
1281                pr_info("Verifying after %s reset...\n", engine->name);
1282                ret = intel_selftest_modify_policy(engine, &saved,
1283                                                   SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1284                if (ret)
1285                        break;
1286
1287                ce = intel_context_create(engine);
1288                if (IS_ERR(ce)) {
1289                        ret = PTR_ERR(ce);
1290                        goto restore;
1291                }
1292
1293                if (!using_guc) {
1294                        ok = verify_wa_lists(gt, lists, "before reset");
1295                        if (!ok) {
1296                                ret = -ESRCH;
1297                                goto err;
1298                        }
1299
1300                        ret = intel_engine_reset(engine, "live_workarounds:idle");
1301                        if (ret) {
1302                                pr_err("%s: Reset failed while idle\n", engine->name);
1303                                goto err;
1304                        }
1305
1306                        ok = verify_wa_lists(gt, lists, "after idle reset");
1307                        if (!ok) {
1308                                ret = -ESRCH;
1309                                goto err;
1310                        }
1311                }
1312
1313                ret = igt_spinner_init(&spin, engine->gt);
1314                if (ret)
1315                        goto err;
1316
1317                rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1318                if (IS_ERR(rq)) {
1319                        ret = PTR_ERR(rq);
1320                        igt_spinner_fini(&spin);
1321                        goto err;
1322                }
1323
1324                ret = request_add_spin(rq, &spin);
1325                if (ret) {
1326                        pr_err("%s: Spinner failed to start\n", engine->name);
1327                        igt_spinner_fini(&spin);
1328                        goto err;
1329                }
1330
1331                /* Ensure the spinner hasn't aborted */
1332                if (i915_request_completed(rq)) {
1333                        ret = -ETIMEDOUT;
1334                        goto skip;
1335                }
1336
1337                if (!using_guc) {
1338                        ret = intel_engine_reset(engine, "live_workarounds:active");
1339                        if (ret) {
1340                                pr_err("%s: Reset failed on an active spinner\n",
1341                                       engine->name);
1342                                igt_spinner_fini(&spin);
1343                                goto err;
1344                        }
1345                }
1346
1347                /* Ensure the reset happens and kills the engine */
1348                if (ret == 0)
1349                        ret = intel_selftest_wait_for_rq(rq);
1350
1351skip:
1352                igt_spinner_end(&spin);
1353                igt_spinner_fini(&spin);
1354
1355                ok = verify_wa_lists(gt, lists, "after busy reset");
1356                if (!ok)
1357                        ret = -ESRCH;
1358
1359err:
1360                intel_context_put(ce);
1361
1362restore:
1363                ret2 = intel_selftest_restore_policy(engine, &saved);
1364                if (ret == 0)
1365                        ret = ret2;
1366                if (ret)
1367                        break;
1368        }
1369
1370        reference_lists_fini(gt, lists);
1371        intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1372        igt_global_reset_unlock(gt);
1373        kfree(lists);
1374
1375        igt_flush_test(gt->i915);
1376
1377        return ret;
1378}
1379
1380int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1381{
1382        static const struct i915_subtest tests[] = {
1383                SUBTEST(live_dirty_whitelist),
1384                SUBTEST(live_reset_whitelist),
1385                SUBTEST(live_isolated_whitelist),
1386                SUBTEST(live_gpu_reset_workarounds),
1387                SUBTEST(live_engine_reset_workarounds),
1388        };
1389
1390        if (intel_gt_is_wedged(&i915->gt))
1391                return 0;
1392
1393        return intel_gt_live_subtests(tests, &i915->gt);
1394}
1395