LXR linux/drivers/gpu/drm/i915/gt/intel_gt

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright © 2019 Intel Corporation
   4 */
   5
   6#include <linux/sched/clock.h>
   7
   8#include "i915_drv.h"
   9#include "i915_irq.h"
  10#include "intel_breadcrumbs.h"
  11#include "intel_gt.h"
  12#include "intel_gt_irq.h"
  13#include "intel_lrc_reg.h"
  14#include "intel_uncore.h"
  15#include "intel_rps.h"
  16
  17static void guc_irq_handler(struct intel_guc *guc, u16 iir)
  18{
  19        if (iir & GUC_INTR_GUC2HOST)
  20                intel_guc_to_host_event_handler(guc);
  21}
  22
  23static u32
  24gen11_gt_engine_identity(struct intel_gt *gt,
  25                         const unsigned int bank, const unsigned int bit)
  26{
  27        void __iomem * const regs = gt->uncore->regs;
  28        u32 timeout_ts;
  29        u32 ident;
  30
  31        lockdep_assert_held(&gt->irq_lock);
  32
  33        raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
  34
  35        /*
  36         * NB: Specs do not specify how long to spin wait,
  37         * so we do ~100us as an educated guess.
  38         */
  39        timeout_ts = (local_clock() >> 10) + 100;
  40        do {
  41                ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
  42        } while (!(ident & GEN11_INTR_DATA_VALID) &&
  43                 !time_after32(local_clock() >> 10, timeout_ts));
  44
  45        if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
  46                DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
  47                          bank, bit, ident);
  48                return 0;
  49        }
  50
  51        raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
  52                      GEN11_INTR_DATA_VALID);
  53
  54        return ident;
  55}
  56
  57static void
  58gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
  59                        const u16 iir)
  60{
  61        if (instance == OTHER_GUC_INSTANCE)
  62                return guc_irq_handler(&gt->uc.guc, iir);
  63
  64        if (instance == OTHER_GTPM_INSTANCE)
  65                return gen11_rps_irq_handler(&gt->rps, iir);
  66
  67        WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
  68                  instance, iir);
  69}
  70
  71static void
  72gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
  73                         const u8 instance, const u16 iir)
  74{
  75        struct intel_engine_cs *engine;
  76
  77        if (instance <= MAX_ENGINE_INSTANCE)
  78                engine = gt->engine_class[class][instance];
  79        else
  80                engine = NULL;
  81
  82        if (likely(engine))
  83                return intel_engine_cs_irq(engine, iir);
  84
  85        WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
  86                  class, instance);
  87}
  88
  89static void
  90gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
  91{
  92        const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
  93        const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
  94        const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
  95
  96        if (unlikely(!intr))
  97                return;
  98
  99        if (class <= COPY_ENGINE_CLASS)
 100                return gen11_engine_irq_handler(gt, class, instance, intr);
 101
 102        if (class == OTHER_CLASS)
 103                return gen11_other_irq_handler(gt, instance, intr);
 104
 105        WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
 106                  class, instance, intr);
 107}
 108
 109static void
 110gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
 111{
 112        void __iomem * const regs = gt->uncore->regs;
 113        unsigned long intr_dw;
 114        unsigned int bit;
 115
 116        lockdep_assert_held(&gt->irq_lock);
 117
 118        intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 119
 120        for_each_set_bit(bit, &intr_dw, 32) {
 121                const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
 122
 123                gen11_gt_identity_handler(gt, ident);
 124        }
 125
 126        /* Clear must be after shared has been served for engine */
 127        raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
 128}
 129
 130void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
 131{
 132        unsigned int bank;
 133
 134        spin_lock(&gt->irq_lock);
 135
 136        for (bank = 0; bank < 2; bank++) {
 137                if (master_ctl & GEN11_GT_DW_IRQ(bank))
 138                        gen11_gt_bank_handler(gt, bank);
 139        }
 140
 141        spin_unlock(&gt->irq_lock);
 142}
 143
 144bool gen11_gt_reset_one_iir(struct intel_gt *gt,
 145                            const unsigned int bank, const unsigned int bit)
 146{
 147        void __iomem * const regs = gt->uncore->regs;
 148        u32 dw;
 149
 150        lockdep_assert_held(&gt->irq_lock);
 151
 152        dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 153        if (dw & BIT(bit)) {
 154                /*
 155                 * According to the BSpec, DW_IIR bits cannot be cleared without
 156                 * first servicing the Selector & Shared IIR registers.
 157                 */
 158                gen11_gt_engine_identity(gt, bank, bit);
 159
 160                /*
 161                 * We locked GT INT DW by reading it. If we want to (try
 162                 * to) recover from this successfully, we need to clear
 163                 * our bit, otherwise we are locking the register for
 164                 * everybody.
 165                 */
 166                raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
 167
 168                return true;
 169        }
 170
 171        return false;
 172}
 173
 174void gen11_gt_irq_reset(struct intel_gt *gt)
 175{
 176        struct intel_uncore *uncore = gt->uncore;
 177
 178        /* Disable RCS, BCS, VCS and VECS class engines. */
 179        intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
 180        intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE,    0);
 181
 182        /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
 183        intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK,   ~0);
 184        intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK,    ~0);
 185        intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK,   ~0);
 186        intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK,   ~0);
 187        if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
 188                intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK,   ~0);
 189        if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
 190                intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK,   ~0);
 191        intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
 192        if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
 193                intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0);
 194
 195        intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 196        intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 197        intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 198        intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 199}
 200
 201void gen11_gt_irq_postinstall(struct intel_gt *gt)
 202{
 203        struct intel_uncore *uncore = gt->uncore;
 204        u32 irqs = GT_RENDER_USER_INTERRUPT;
 205        u32 dmask;
 206        u32 smask;
 207
 208        if (!intel_uc_wants_guc_submission(&gt->uc))
 209                irqs |= GT_CS_MASTER_ERROR_INTERRUPT |
 210                        GT_CONTEXT_SWITCH_INTERRUPT |
 211                        GT_WAIT_SEMAPHORE_INTERRUPT;
 212
 213        dmask = irqs << 16 | irqs;
 214        smask = irqs << 16;
 215
 216        BUILD_BUG_ON(irqs & 0xffff0000);
 217
 218        /* Enable RCS, BCS, VCS and VECS class interrupts. */
 219        intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
 220        intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
 221
 222        /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
 223        intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
 224        intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
 225        intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
 226        intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
 227        if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
 228                intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask);
 229        if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
 230                intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask);
 231        intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
 232        if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
 233                intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask);
 234        /*
 235         * RPS interrupts will get enabled/disabled on demand when RPS itself
 236         * is enabled/disabled.
 237         */
 238        gt->pm_ier = 0x0;
 239        gt->pm_imr = ~gt->pm_ier;
 240        intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 241        intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 242
 243        /* Same thing for GuC interrupts */
 244        intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 245        intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 246}
 247
 248void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 249{
 250        if (gt_iir & GT_RENDER_USER_INTERRUPT)
 251                intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
 252                                    gt_iir);
 253
 254        if (gt_iir & ILK_BSD_USER_INTERRUPT)
 255                intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
 256                                    gt_iir);
 257}
 258
 259static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
 260{
 261        if (!HAS_L3_DPF(gt->i915))
 262                return;
 263
 264        spin_lock(&gt->irq_lock);
 265        gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
 266        spin_unlock(&gt->irq_lock);
 267
 268        if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
 269                gt->i915->l3_parity.which_slice |= 1 << 1;
 270
 271        if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
 272                gt->i915->l3_parity.which_slice |= 1 << 0;
 273
 274        schedule_work(&gt->i915->l3_parity.error_work);
 275}
 276
 277void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 278{
 279        if (gt_iir & GT_RENDER_USER_INTERRUPT)
 280                intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
 281                                    gt_iir);
 282
 283        if (gt_iir & GT_BSD_USER_INTERRUPT)
 284                intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
 285                                    gt_iir >> 12);
 286
 287        if (gt_iir & GT_BLT_USER_INTERRUPT)
 288                intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
 289                                    gt_iir >> 22);
 290
 291        if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 292                      GT_BSD_CS_ERROR_INTERRUPT |
 293                      GT_CS_MASTER_ERROR_INTERRUPT))
 294                DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
 295
 296        if (gt_iir & GT_PARITY_ERROR(gt->i915))
 297                gen7_parity_error_irq_handler(gt, gt_iir);
 298}
 299
 300void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
 301{
 302        void __iomem * const regs = gt->uncore->regs;
 303        u32 iir;
 304
 305        if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
 306                iir = raw_reg_read(regs, GEN8_GT_IIR(0));
 307                if (likely(iir)) {
 308                        intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
 309                                            iir >> GEN8_RCS_IRQ_SHIFT);
 310                        intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
 311                                            iir >> GEN8_BCS_IRQ_SHIFT);
 312                        raw_reg_write(regs, GEN8_GT_IIR(0), iir);
 313                }
 314        }
 315
 316        if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
 317                iir = raw_reg_read(regs, GEN8_GT_IIR(1));
 318                if (likely(iir)) {
 319                        intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
 320                                            iir >> GEN8_VCS0_IRQ_SHIFT);
 321                        intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1],
 322                                            iir >> GEN8_VCS1_IRQ_SHIFT);
 323                        raw_reg_write(regs, GEN8_GT_IIR(1), iir);
 324                }
 325        }
 326
 327        if (master_ctl & GEN8_GT_VECS_IRQ) {
 328                iir = raw_reg_read(regs, GEN8_GT_IIR(3));
 329                if (likely(iir)) {
 330                        intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
 331                                            iir >> GEN8_VECS_IRQ_SHIFT);
 332                        raw_reg_write(regs, GEN8_GT_IIR(3), iir);
 333                }
 334        }
 335
 336        if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
 337                iir = raw_reg_read(regs, GEN8_GT_IIR(2));
 338                if (likely(iir)) {
 339                        gen6_rps_irq_handler(&gt->rps, iir);
 340                        guc_irq_handler(&gt->uc.guc, iir >> 16);
 341                        raw_reg_write(regs, GEN8_GT_IIR(2), iir);
 342                }
 343        }
 344}
 345
 346void gen8_gt_irq_reset(struct intel_gt *gt)
 347{
 348        struct intel_uncore *uncore = gt->uncore;
 349
 350        GEN8_IRQ_RESET_NDX(uncore, GT, 0);
 351        GEN8_IRQ_RESET_NDX(uncore, GT, 1);
 352        GEN8_IRQ_RESET_NDX(uncore, GT, 2);
 353        GEN8_IRQ_RESET_NDX(uncore, GT, 3);
 354}
 355
 356void gen8_gt_irq_postinstall(struct intel_gt *gt)
 357{
 358        /* These are interrupts we'll toggle with the ring mask register */
 359        const u32 irqs =
 360                GT_CS_MASTER_ERROR_INTERRUPT |
 361                GT_RENDER_USER_INTERRUPT |
 362                GT_CONTEXT_SWITCH_INTERRUPT |
 363                GT_WAIT_SEMAPHORE_INTERRUPT;
 364        const u32 gt_interrupts[] = {
 365                irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
 366                irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
 367                0,
 368                irqs << GEN8_VECS_IRQ_SHIFT,
 369        };
 370        struct intel_uncore *uncore = gt->uncore;
 371
 372        gt->pm_ier = 0x0;
 373        gt->pm_imr = ~gt->pm_ier;
 374        GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
 375        GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
 376        /*
 377         * RPS interrupts will get enabled/disabled on demand when RPS itself
 378         * is enabled/disabled. Same wil be the case for GuC interrupts.
 379         */
 380        GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
 381        GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
 382}
 383
 384static void gen5_gt_update_irq(struct intel_gt *gt,
 385                               u32 interrupt_mask,
 386                               u32 enabled_irq_mask)
 387{
 388        lockdep_assert_held(&gt->irq_lock);
 389
 390        GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
 391
 392        gt->gt_imr &= ~interrupt_mask;
 393        gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
 394        intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
 395}
 396
 397void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
 398{
 399        gen5_gt_update_irq(gt, mask, mask);
 400        intel_uncore_posting_read_fw(gt->uncore, GTIMR);
 401}
 402
 403void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
 404{
 405        gen5_gt_update_irq(gt, mask, 0);
 406}
 407
 408void gen5_gt_irq_reset(struct intel_gt *gt)
 409{
 410        struct intel_uncore *uncore = gt->uncore;
 411
 412        GEN3_IRQ_RESET(uncore, GT);
 413        if (GRAPHICS_VER(gt->i915) >= 6)
 414                GEN3_IRQ_RESET(uncore, GEN6_PM);
 415}
 416
 417void gen5_gt_irq_postinstall(struct intel_gt *gt)
 418{
 419        struct intel_uncore *uncore = gt->uncore;
 420        u32 pm_irqs = 0;
 421        u32 gt_irqs = 0;
 422
 423        gt->gt_imr = ~0;
 424        if (HAS_L3_DPF(gt->i915)) {
 425                /* L3 parity interrupt is always unmasked. */
 426                gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
 427                gt_irqs |= GT_PARITY_ERROR(gt->i915);
 428        }
 429
 430        gt_irqs |= GT_RENDER_USER_INTERRUPT;
 431        if (GRAPHICS_VER(gt->i915) == 5)
 432                gt_irqs |= ILK_BSD_USER_INTERRUPT;
 433        else
 434                gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
 435
 436        GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
 437
 438        if (GRAPHICS_VER(gt->i915) >= 6) {
 439                /*
 440                 * RPS interrupts will get enabled/disabled on demand when RPS
 441                 * itself is enabled/disabled.
 442                 */
 443                if (HAS_ENGINE(gt, VECS0)) {
 444                        pm_irqs |= PM_VEBOX_USER_INTERRUPT;
 445                        gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
 446                }
 447
 448                gt->pm_imr = 0xffffffff;
 449                GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
 450        }
 451}
 452