drivers/gpu/drm/i915/gt/intel_gt_irq.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/sched/clock.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "intel_gt.h"
  12 #include "intel_gt_irq.h"
  13 #include "intel_uncore.h"
  14 #include "intel_rps.h"
  15
  16 static void guc_irq_handler(struct intel_guc *guc, u16 iir)
  17 {
  18         if (iir & GUC_INTR_GUC2HOST)
  19                 intel_guc_to_host_event_handler(guc);
  20 }
  21
  22 static void
  23 cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
  24 {
  25         bool tasklet = false;
  26
  27         if (unlikely(iir & GT_CS_MASTER_ERROR_INTERRUPT)) {
  28                 u32 eir;
  29
  30                 eir = ENGINE_READ(engine, RING_EIR);
  31                 ENGINE_TRACE(engine, "CS error: %x\n", eir);
  32
  33                 /* Disable the error interrupt until after the reset */
  34                 if (likely(eir)) {
  35                         ENGINE_WRITE(engine, RING_EMR, ~0u);
  36                         ENGINE_WRITE(engine, RING_EIR, eir);
  37                         WRITE_ONCE(engine->execlists.error_interrupt, eir);
  38                         tasklet = true;
  39                 }
  40         }
  41
  42         if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
  43                 tasklet = true;
  44
  45         if (iir & GT_RENDER_USER_INTERRUPT) {
  46                 intel_engine_signal_breadcrumbs(engine);
  47                 tasklet |= intel_engine_needs_breadcrumb_tasklet(engine);
  48         }
  49
  50         if (tasklet)
  51                 tasklet_hi_schedule(&engine->execlists.tasklet);
  52 }
  53
  54 static u32
  55 gen11_gt_engine_identity(struct intel_gt *gt,
  56                          const unsigned int bank, const unsigned int bit)
  57 {
  58         void __iomem * const regs = gt->uncore->regs;
  59         u32 timeout_ts;
  60         u32 ident;
  61
  62         lockdep_assert_held(&gt->irq_lock);
  63
  64         raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
  65
  66         /*
  67          * NB: Specs do not specify how long to spin wait,
  68          * so we do ~100us as an educated guess.
  69          */
  70         timeout_ts = (local_clock() >> 10) + 100;
  71         do {
  72                 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
  73         } while (!(ident & GEN11_INTR_DATA_VALID) &&
  74                  !time_after32(local_clock() >> 10, timeout_ts));
  75
  76         if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
  77                 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
  78                           bank, bit, ident);
  79                 return 0;
  80         }
  81
  82         raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
  83                       GEN11_INTR_DATA_VALID);
  84
  85         return ident;
  86 }
  87
  88 static void
  89 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
  90                         const u16 iir)
  91 {
  92         if (instance == OTHER_GUC_INSTANCE)
  93                 return guc_irq_handler(&gt->uc.guc, iir);
  94
  95         if (instance == OTHER_GTPM_INSTANCE)
  96                 return gen11_rps_irq_handler(&gt->rps, iir);
  97
  98         WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
  99                   instance, iir);
 100 }
 101
 102 static void
 103 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
 104                          const u8 instance, const u16 iir)
 105 {
 106         struct intel_engine_cs *engine;
 107
 108         if (instance <= MAX_ENGINE_INSTANCE)
 109                 engine = gt->engine_class[class][instance];
 110         else
 111                 engine = NULL;
 112
 113         if (likely(engine))
 114                 return cs_irq_handler(engine, iir);
 115
 116         WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
 117                   class, instance);
 118 }
 119
 120 static void
 121 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
 122 {
 123         const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
 124         const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
 125         const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
 126
 127         if (unlikely(!intr))
 128                 return;
 129
 130         if (class <= COPY_ENGINE_CLASS)
 131                 return gen11_engine_irq_handler(gt, class, instance, intr);
 132
 133         if (class == OTHER_CLASS)
 134                 return gen11_other_irq_handler(gt, instance, intr);
 135
 136         WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
 137                   class, instance, intr);
 138 }
 139
 140 static void
 141 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
 142 {
 143         void __iomem * const regs = gt->uncore->regs;
 144         unsigned long intr_dw;
 145         unsigned int bit;
 146
 147         lockdep_assert_held(&gt->irq_lock);
 148
 149         intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 150
 151         for_each_set_bit(bit, &intr_dw, 32) {
 152                 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
 153
 154                 gen11_gt_identity_handler(gt, ident);
 155         }
 156
 157         /* Clear must be after shared has been served for engine */
 158         raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
 159 }
 160
 161 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
 162 {
 163         unsigned int bank;
 164
 165         spin_lock(&gt->irq_lock);
 166
 167         for (bank = 0; bank < 2; bank++) {
 168                 if (master_ctl & GEN11_GT_DW_IRQ(bank))
 169                         gen11_gt_bank_handler(gt, bank);
 170         }
 171
 172         spin_unlock(&gt->irq_lock);
 173 }
 174
 175 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
 176                             const unsigned int bank, const unsigned int bit)
 177 {
 178         void __iomem * const regs = gt->uncore->regs;
 179         u32 dw;
 180
 181         lockdep_assert_held(&gt->irq_lock);
 182
 183         dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 184         if (dw & BIT(bit)) {
 185                 /*
 186                  * According to the BSpec, DW_IIR bits cannot be cleared without
 187                  * first servicing the Selector & Shared IIR registers.
 188                  */
 189                 gen11_gt_engine_identity(gt, bank, bit);
 190
 191                 /*
 192                  * We locked GT INT DW by reading it. If we want to (try
 193                  * to) recover from this successfully, we need to clear
 194                  * our bit, otherwise we are locking the register for
 195                  * everybody.
 196                  */
 197                 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
 198
 199                 return true;
 200         }
 201
 202         return false;
 203 }
 204
 205 void gen11_gt_irq_reset(struct intel_gt *gt)
 206 {
 207         struct intel_uncore *uncore = gt->uncore;
 208
 209         /* Disable RCS, BCS, VCS and VECS class engines. */
 210         intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
 211         intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE,    0);
 212
 213         /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
 214         intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK,   ~0);
 215         intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK,    ~0);
 216         intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK,   ~0);
 217         intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK,   ~0);
 218         intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
 219
 220         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 221         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 222         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 223         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 224 }
 225
 226 void gen11_gt_irq_postinstall(struct intel_gt *gt)
 227 {
 228         const u32 irqs =
 229                 GT_CS_MASTER_ERROR_INTERRUPT |
 230                 GT_RENDER_USER_INTERRUPT |
 231                 GT_CONTEXT_SWITCH_INTERRUPT;
 232         struct intel_uncore *uncore = gt->uncore;
 233         const u32 dmask = irqs << 16 | irqs;
 234         const u32 smask = irqs << 16;
 235
 236         BUILD_BUG_ON(irqs & 0xffff0000);
 237
 238         /* Enable RCS, BCS, VCS and VECS class interrupts. */
 239         intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
 240         intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
 241
 242         /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
 243         intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
 244         intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
 245         intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
 246         intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
 247         intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
 248
 249         /*
 250          * RPS interrupts will get enabled/disabled on demand when RPS itself
 251          * is enabled/disabled.
 252          */
 253         gt->pm_ier = 0x0;
 254         gt->pm_imr = ~gt->pm_ier;
 255         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
 256         intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 257
 258         /* Same thing for GuC interrupts */
 259         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 260         intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK,  ~0);
 261 }
 262
 263 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 264 {
 265         if (gt_iir & GT_RENDER_USER_INTERRUPT)
 266                 intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]);
 267         if (gt_iir & ILK_BSD_USER_INTERRUPT)
 268                 intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]);
 269 }
 270
 271 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
 272 {
 273         if (!HAS_L3_DPF(gt->i915))
 274                 return;
 275
 276         spin_lock(&gt->irq_lock);
 277         gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
 278         spin_unlock(&gt->irq_lock);
 279
 280         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
 281                 gt->i915->l3_parity.which_slice |= 1 << 1;
 282
 283         if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
 284                 gt->i915->l3_parity.which_slice |= 1 << 0;
 285
 286         schedule_work(&gt->i915->l3_parity.error_work);
 287 }
 288
 289 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
 290 {
 291         if (gt_iir & GT_RENDER_USER_INTERRUPT)
 292                 intel_engine_signal_breadcrumbs(gt->engine_class[RENDER_CLASS][0]);
 293         if (gt_iir & GT_BSD_USER_INTERRUPT)
 294                 intel_engine_signal_breadcrumbs(gt->engine_class[VIDEO_DECODE_CLASS][0]);
 295         if (gt_iir & GT_BLT_USER_INTERRUPT)
 296                 intel_engine_signal_breadcrumbs(gt->engine_class[COPY_ENGINE_CLASS][0]);
 297
 298         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 299                       GT_BSD_CS_ERROR_INTERRUPT |
 300                       GT_CS_MASTER_ERROR_INTERRUPT))
 301                 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
 302
 303         if (gt_iir & GT_PARITY_ERROR(gt->i915))
 304                 gen7_parity_error_irq_handler(gt, gt_iir);
 305 }
 306
 307 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
 308 {
 309         void __iomem * const regs = gt->uncore->regs;
 310         u32 iir;
 311
 312         if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
 313                 iir = raw_reg_read(regs, GEN8_GT_IIR(0));
 314                 if (likely(iir)) {
 315                         cs_irq_handler(gt->engine_class[RENDER_CLASS][0],
 316                                        iir >> GEN8_RCS_IRQ_SHIFT);
 317                         cs_irq_handler(gt->engine_class[COPY_ENGINE_CLASS][0],
 318                                        iir >> GEN8_BCS_IRQ_SHIFT);
 319                         raw_reg_write(regs, GEN8_GT_IIR(0), iir);
 320                 }
 321         }
 322
 323         if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
 324                 iir = raw_reg_read(regs, GEN8_GT_IIR(1));
 325                 if (likely(iir)) {
 326                         cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][0],
 327                                        iir >> GEN8_VCS0_IRQ_SHIFT);
 328                         cs_irq_handler(gt->engine_class[VIDEO_DECODE_CLASS][1],
 329                                        iir >> GEN8_VCS1_IRQ_SHIFT);
 330                         raw_reg_write(regs, GEN8_GT_IIR(1), iir);
 331                 }
 332         }
 333
 334         if (master_ctl & GEN8_GT_VECS_IRQ) {
 335                 iir = raw_reg_read(regs, GEN8_GT_IIR(3));
 336                 if (likely(iir)) {
 337                         cs_irq_handler(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
 338                                        iir >> GEN8_VECS_IRQ_SHIFT);
 339                         raw_reg_write(regs, GEN8_GT_IIR(3), iir);
 340                 }
 341         }
 342
 343         if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
 344                 iir = raw_reg_read(regs, GEN8_GT_IIR(2));
 345                 if (likely(iir)) {
 346                         gen6_rps_irq_handler(&gt->rps, iir);
 347                         guc_irq_handler(&gt->uc.guc, iir >> 16);
 348                         raw_reg_write(regs, GEN8_GT_IIR(2), iir);
 349                 }
 350         }
 351 }
 352
 353 void gen8_gt_irq_reset(struct intel_gt *gt)
 354 {
 355         struct intel_uncore *uncore = gt->uncore;
 356
 357         GEN8_IRQ_RESET_NDX(uncore, GT, 0);
 358         GEN8_IRQ_RESET_NDX(uncore, GT, 1);
 359         GEN8_IRQ_RESET_NDX(uncore, GT, 2);
 360         GEN8_IRQ_RESET_NDX(uncore, GT, 3);
 361 }
 362
 363 void gen8_gt_irq_postinstall(struct intel_gt *gt)
 364 {
 365         /* These are interrupts we'll toggle with the ring mask register */
 366         const u32 irqs =
 367                 GT_CS_MASTER_ERROR_INTERRUPT |
 368                 GT_RENDER_USER_INTERRUPT |
 369                 GT_CONTEXT_SWITCH_INTERRUPT;
 370         const u32 gt_interrupts[] = {
 371                 irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
 372                 irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
 373                 0,
 374                 irqs << GEN8_VECS_IRQ_SHIFT,
 375         };
 376         struct intel_uncore *uncore = gt->uncore;
 377
 378         gt->pm_ier = 0x0;
 379         gt->pm_imr = ~gt->pm_ier;
 380         GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
 381         GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
 382         /*
 383          * RPS interrupts will get enabled/disabled on demand when RPS itself
 384          * is enabled/disabled. Same wil be the case for GuC interrupts.
 385          */
 386         GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
 387         GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
 388 }
 389
 390 static void gen5_gt_update_irq(struct intel_gt *gt,
 391                                u32 interrupt_mask,
 392                                u32 enabled_irq_mask)
 393 {
 394         lockdep_assert_held(&gt->irq_lock);
 395
 396         GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
 397
 398         gt->gt_imr &= ~interrupt_mask;
 399         gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
 400         intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
 401 }
 402
 403 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
 404 {
 405         gen5_gt_update_irq(gt, mask, mask);
 406         intel_uncore_posting_read_fw(gt->uncore, GTIMR);
 407 }
 408
 409 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
 410 {
 411         gen5_gt_update_irq(gt, mask, 0);
 412 }
 413
 414 void gen5_gt_irq_reset(struct intel_gt *gt)
 415 {
 416         struct intel_uncore *uncore = gt->uncore;
 417
 418         GEN3_IRQ_RESET(uncore, GT);
 419         if (INTEL_GEN(gt->i915) >= 6)
 420                 GEN3_IRQ_RESET(uncore, GEN6_PM);
 421 }
 422
 423 void gen5_gt_irq_postinstall(struct intel_gt *gt)
 424 {
 425         struct intel_uncore *uncore = gt->uncore;
 426         u32 pm_irqs = 0;
 427         u32 gt_irqs = 0;
 428
 429         gt->gt_imr = ~0;
 430         if (HAS_L3_DPF(gt->i915)) {
 431                 /* L3 parity interrupt is always unmasked. */
 432                 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
 433                 gt_irqs |= GT_PARITY_ERROR(gt->i915);
 434         }
 435
 436         gt_irqs |= GT_RENDER_USER_INTERRUPT;
 437         if (IS_GEN(gt->i915, 5))
 438                 gt_irqs |= ILK_BSD_USER_INTERRUPT;
 439         else
 440                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
 441
 442         GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
 443
 444         if (INTEL_GEN(gt->i915) >= 6) {
 445                 /*
 446                  * RPS interrupts will get enabled/disabled on demand when RPS
 447                  * itself is enabled/disabled.
 448                  */
 449                 if (HAS_ENGINE(gt->i915, VECS0)) {
 450                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
 451                         gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
 452                 }
 453
 454                 gt->pm_imr = 0xffffffff;
 455                 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
 456         }
 457 }