drivers/gpu/drm/i915/intel_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_workarounds.h"
   9
  10 /**
  11  * DOC: Hardware workarounds
  12  *
  13  * This file is intended as a central place to implement most [1]_ of the
  14  * required workarounds for hardware to work as originally intended. They fall
  15  * in five basic categories depending on how/when they are applied:
  16  *
  17  * - Workarounds that touch registers that are saved/restored to/from the HW
  18  *   context image. The list is emitted (via Load Register Immediate commands)
  19  *   everytime a new context is created.
  20  * - GT workarounds. The list of these WAs is applied whenever these registers
  21  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  22  * - Display workarounds. The list is applied during display clock-gating
  23  *   initialization.
  24  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  25  *   them directly. This is just a special case of a MMMIO workaround (as we
  26  *   write the list of these to/be-whitelisted registers to some special HW
  27  *   registers).
  28  * - Workaround batchbuffers, that get executed automatically by the hardware
  29  *   on every HW context restore.
  30  *
  31  * .. [1] Please notice that there are other WAs that, due to their nature,
  32  *    cannot be applied from a central place. Those are peppered around the rest
  33  *    of the code, as needed.
  34  *
  35  * .. [2] Technically, some registers are powercontext saved & restored, so they
  36  *    survive a suspend/resume. In practice, writing them again is not too
  37  *    costly and simplifies things. We can revisit this in the future.
  38  *
  39  * Layout
  40  * ''''''
  41  *
  42  * Keep things in this file ordered by WA type, as per the above (context, GT,
  43  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  44  * following order:
  45  *
  46  * - Infrastructure functions and macros
  47  * - WAs per platform in standard gen/chrono order
  48  * - Public functions to init or apply the given workaround type.
  49  */
  50
  51 static void wa_init_start(struct i915_wa_list *wal, const char *name)
  52 {
  53         wal->name = name;
  54 }
  55
  56 #define WA_LIST_CHUNK (1 << 4)
  57
  58 static void wa_init_finish(struct i915_wa_list *wal)
  59 {
  60         /* Trim unused entries. */
  61         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  62                 struct i915_wa *list = kmemdup(wal->list,
  63                                                wal->count * sizeof(*list),
  64                                                GFP_KERNEL);
  65
  66                 if (list) {
  67                         kfree(wal->list);
  68                         wal->list = list;
  69                 }
  70         }
  71
  72         if (!wal->count)
  73                 return;
  74
  75         DRM_DEBUG_DRIVER("Initialized %u %s workarounds\n",
  76                          wal->wa_count, wal->name);
  77 }
  78
  79 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
  80 {
  81         unsigned int addr = i915_mmio_reg_offset(wa->reg);
  82         unsigned int start = 0, end = wal->count;
  83         const unsigned int grow = WA_LIST_CHUNK;
  84         struct i915_wa *wa_;
  85
  86         GEM_BUG_ON(!is_power_of_2(grow));
  87
  88         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
  89                 struct i915_wa *list;
  90
  91                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
  92                                      GFP_KERNEL);
  93                 if (!list) {
  94                         DRM_ERROR("No space for workaround init!\n");
  95                         return;
  96                 }
  97
  98                 if (wal->list)
  99                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 100
 101                 wal->list = list;
 102         }
 103
 104         while (start < end) {
 105                 unsigned int mid = start + (end - start) / 2;
 106
 107                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 108                         start = mid + 1;
 109                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 110                         end = mid;
 111                 } else {
 112                         wa_ = &wal->list[mid];
 113
 114                         if ((wa->mask & ~wa_->mask) == 0) {
 115                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (mask: %08x, value: %08x)\n",
 116                                           i915_mmio_reg_offset(wa_->reg),
 117                                           wa_->mask, wa_->val);
 118
 119                                 wa_->val &= ~wa->mask;
 120                         }
 121
 122                         wal->wa_count++;
 123                         wa_->val |= wa->val;
 124                         wa_->mask |= wa->mask;
 125                         return;
 126                 }
 127         }
 128
 129         wal->wa_count++;
 130         wa_ = &wal->list[wal->count++];
 131         *wa_ = *wa;
 132
 133         while (wa_-- > wal->list) {
 134                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 135                            i915_mmio_reg_offset(wa_[1].reg));
 136                 if (i915_mmio_reg_offset(wa_[1].reg) >
 137                     i915_mmio_reg_offset(wa_[0].reg))
 138                         break;
 139
 140                 swap(wa_[1], wa_[0]);
 141         }
 142 }
 143
 144 static void
 145 __wa_add(struct i915_wa_list *wal, i915_reg_t reg, u32 mask, u32 val)
 146 {
 147         struct i915_wa wa = {
 148                 .reg = reg,
 149                 .mask = mask,
 150                 .val = val
 151         };
 152
 153         _wa_add(wal, &wa);
 154 }
 155
 156 #define WA_REG(addr, mask, val) __wa_add(wal, (addr), (mask), (val))
 157
 158 #define WA_SET_BIT_MASKED(addr, mask) \
 159         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
 160
 161 #define WA_CLR_BIT_MASKED(addr, mask) \
 162         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
 163
 164 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 165         WA_REG(addr, (mask), _MASKED_FIELD(mask, value))
 166
 167 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine)
 168 {
 169         struct i915_wa_list *wal = &engine->ctx_wa_list;
 170
 171         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 172
 173         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 174         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 175
 176         /* WaDisablePartialInstShootdown:bdw,chv */
 177         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 178                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 179
 180         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 181          * workaround for for a possible hang in the unlikely event a TLB
 182          * invalidation occurs during a PSD flush.
 183          */
 184         /* WaForceEnableNonCoherent:bdw,chv */
 185         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 186         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 187                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 188                           HDC_FORCE_NON_COHERENT);
 189
 190         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 191          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 192          *  polygons in the same 8x4 pixel/sample area to be processed without
 193          *  stalling waiting for the earlier ones to write to Hierarchical Z
 194          *  buffer."
 195          *
 196          * This optimization is off by default for BDW and CHV; turn it on.
 197          */
 198         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 199
 200         /* Wa4x4STCOptimizationDisable:bdw,chv */
 201         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 202
 203         /*
 204          * BSpec recommends 8x4 when MSAA is used,
 205          * however in practice 16x4 seems fastest.
 206          *
 207          * Note that PS/WM thread counts depend on the WIZ hashing
 208          * disable bit, which we don't touch here, but it's good
 209          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 210          */
 211         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 212                             GEN6_WIZ_HASHING_MASK,
 213                             GEN6_WIZ_HASHING_16x4);
 214 }
 215
 216 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine)
 217 {
 218         struct drm_i915_private *i915 = engine->i915;
 219         struct i915_wa_list *wal = &engine->ctx_wa_list;
 220
 221         gen8_ctx_workarounds_init(engine);
 222
 223         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 224         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 225
 226         /* WaDisableDopClockGating:bdw
 227          *
 228          * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
 229          * to disable EUTC clock gating.
 230          */
 231         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 232                           DOP_CLOCK_GATING_DISABLE);
 233
 234         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 235                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 236
 237         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 238                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 239                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 240                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 241                           (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 242 }
 243
 244 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine)
 245 {
 246         struct i915_wa_list *wal = &engine->ctx_wa_list;
 247
 248         gen8_ctx_workarounds_init(engine);
 249
 250         /* WaDisableThreadStallDopClockGating:chv */
 251         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 252
 253         /* Improve HiZ throughput on CHV. */
 254         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 255 }
 256
 257 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine)
 258 {
 259         struct drm_i915_private *i915 = engine->i915;
 260         struct i915_wa_list *wal = &engine->ctx_wa_list;
 261
 262         if (HAS_LLC(i915)) {
 263                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 264                  *
 265                  * Must match Display Engine. See
 266                  * WaCompressedResourceDisplayNewHashMode.
 267                  */
 268                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 269                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
 270                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 271                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 272         }
 273
 274         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 275         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 276         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 277                           FLOW_CONTROL_ENABLE |
 278                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 279
 280         /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
 281         if (!IS_COFFEELAKE(i915))
 282                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 283                                   GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 284
 285         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 286         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 287         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 288                           GEN9_ENABLE_YV12_BUGFIX |
 289                           GEN9_ENABLE_GPGPU_PREEMPTION);
 290
 291         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 292         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 293         WA_SET_BIT_MASKED(CACHE_MODE_1,
 294                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 295                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 296
 297         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 298         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 299                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 300
 301         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 302         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 303                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 304                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 305
 306         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 307          * both tied to WaForceContextSaveRestoreNonCoherent
 308          * in some hsds for skl. We keep the tie for all gen9. The
 309          * documentation is a bit hazy and so we want to get common behaviour,
 310          * even though there is no clear evidence we would need both on kbl/bxt.
 311          * This area has been source of system hangs so we play it safe
 312          * and mimic the skl regardless of what bspec says.
 313          *
 314          * Use Force Non-Coherent whenever executing a 3D context. This
 315          * is a workaround for a possible hang in the unlikely event
 316          * a TLB invalidation occurs during a PSD flush.
 317          */
 318
 319         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 320         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 321                           HDC_FORCE_NON_COHERENT);
 322
 323         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 324         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915))
 325                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 326                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 327
 328         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 329         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 330
 331         /*
 332          * Supporting preemption with fine-granularity requires changes in the
 333          * batch buffer programming. Since we can't break old userspace, we
 334          * need to set our default preemption level to safe value. Userspace is
 335          * still able to use more fine-grained preemption levels, since in
 336          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 337          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 338          * not real HW workarounds, but merely a way to start using preemption
 339          * while maintaining old contract with userspace.
 340          */
 341
 342         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 343         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 344
 345         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 346         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 347                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 348                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 349
 350         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 351         if (IS_GEN9_LP(i915))
 352                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 353 }
 354
 355 static void skl_tune_iz_hashing(struct intel_engine_cs *engine)
 356 {
 357         struct drm_i915_private *i915 = engine->i915;
 358         struct i915_wa_list *wal = &engine->ctx_wa_list;
 359         u8 vals[3] = { 0, 0, 0 };
 360         unsigned int i;
 361
 362         for (i = 0; i < 3; i++) {
 363                 u8 ss;
 364
 365                 /*
 366                  * Only consider slices where one, and only one, subslice has 7
 367                  * EUs
 368                  */
 369                 if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]))
 370                         continue;
 371
 372                 /*
 373                  * subslice_7eu[i] != 0 (because of the check above) and
 374                  * ss_max == 4 (maximum number of subslices possible per slice)
 375                  *
 376                  * ->    0 <= ss <= 3;
 377                  */
 378                 ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1;
 379                 vals[i] = 3 - ss;
 380         }
 381
 382         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 383                 return;
 384
 385         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 386         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 387                             GEN9_IZ_HASHING_MASK(2) |
 388                             GEN9_IZ_HASHING_MASK(1) |
 389                             GEN9_IZ_HASHING_MASK(0),
 390                             GEN9_IZ_HASHING(2, vals[2]) |
 391                             GEN9_IZ_HASHING(1, vals[1]) |
 392                             GEN9_IZ_HASHING(0, vals[0]));
 393 }
 394
 395 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine)
 396 {
 397         gen9_ctx_workarounds_init(engine);
 398         skl_tune_iz_hashing(engine);
 399 }
 400
 401 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine)
 402 {
 403         struct i915_wa_list *wal = &engine->ctx_wa_list;
 404
 405         gen9_ctx_workarounds_init(engine);
 406
 407         /* WaDisableThreadStallDopClockGating:bxt */
 408         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 409                           STALL_DOP_GATING_DISABLE);
 410
 411         /* WaToEnableHwFixForPushConstHWBug:bxt */
 412         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 413                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 414 }
 415
 416 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine)
 417 {
 418         struct drm_i915_private *i915 = engine->i915;
 419         struct i915_wa_list *wal = &engine->ctx_wa_list;
 420
 421         gen9_ctx_workarounds_init(engine);
 422
 423         /* WaToEnableHwFixForPushConstHWBug:kbl */
 424         if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
 425                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 426                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 427
 428         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 429         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 430                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 431 }
 432
 433 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine)
 434 {
 435         struct i915_wa_list *wal = &engine->ctx_wa_list;
 436
 437         gen9_ctx_workarounds_init(engine);
 438
 439         /* WaToEnableHwFixForPushConstHWBug:glk */
 440         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 441                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 442 }
 443
 444 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine)
 445 {
 446         struct i915_wa_list *wal = &engine->ctx_wa_list;
 447
 448         gen9_ctx_workarounds_init(engine);
 449
 450         /* WaToEnableHwFixForPushConstHWBug:cfl */
 451         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 452                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 453
 454         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 455         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 456                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 457 }
 458
 459 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine)
 460 {
 461         struct drm_i915_private *i915 = engine->i915;
 462         struct i915_wa_list *wal = &engine->ctx_wa_list;
 463
 464         /* WaForceContextSaveRestoreNonCoherent:cnl */
 465         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
 466                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 467
 468         /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
 469         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 470                 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
 471
 472         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 473         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 474                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 475
 476         /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
 477         if (IS_CNL_REVID(i915, 0, CNL_REVID_B0))
 478                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 479                                   GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
 480
 481         /* WaPushConstantDereferenceHoldDisable:cnl */
 482         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 483
 484         /* FtrEnableFastAnisoL1BankingFix:cnl */
 485         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 486
 487         /* WaDisable3DMidCmdPreemption:cnl */
 488         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 489
 490         /* WaDisableGPGPUMidCmdPreemption:cnl */
 491         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 492                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 493                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 494
 495         /* WaDisableEarlyEOT:cnl */
 496         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 497 }
 498
 499 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine)
 500 {
 501         struct drm_i915_private *i915 = engine->i915;
 502         struct i915_wa_list *wal = &engine->ctx_wa_list;
 503
 504         /* Wa_1604370585:icl (pre-prod)
 505          * Formerly known as WaPushConstantDereferenceHoldDisable
 506          */
 507         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 508                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 509                                   PUSH_CONSTANT_DEREF_DISABLE);
 510
 511         /* WaForceEnableNonCoherent:icl
 512          * This is not the same workaround as in early Gen9 platforms, where
 513          * lacking this could cause system hangs, but coherency performance
 514          * overhead is high and only a few compute workloads really need it
 515          * (the register is whitelisted in hardware now, so UMDs can opt in
 516          * for coherency if they have a good reason).
 517          */
 518         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 519
 520         /* Wa_2006611047:icl (pre-prod)
 521          * Formerly known as WaDisableImprovedTdlClkGating
 522          */
 523         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 524                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 525                                   GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 526
 527         /* WaEnableStateCacheRedirectToCS:icl */
 528         WA_SET_BIT_MASKED(GEN9_SLICE_COMMON_ECO_CHICKEN1,
 529                           GEN11_STATE_CACHE_REDIRECT_TO_CS);
 530
 531         /* Wa_2006665173:icl (pre-prod) */
 532         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 533                 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 534                                   GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 535 }
 536
 537 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 538 {
 539         struct drm_i915_private *i915 = engine->i915;
 540         struct i915_wa_list *wal = &engine->ctx_wa_list;
 541
 542         wa_init_start(wal, "context");
 543
 544         if (INTEL_GEN(i915) < 8)
 545                 return;
 546         else if (IS_BROADWELL(i915))
 547                 bdw_ctx_workarounds_init(engine);
 548         else if (IS_CHERRYVIEW(i915))
 549                 chv_ctx_workarounds_init(engine);
 550         else if (IS_SKYLAKE(i915))
 551                 skl_ctx_workarounds_init(engine);
 552         else if (IS_BROXTON(i915))
 553                 bxt_ctx_workarounds_init(engine);
 554         else if (IS_KABYLAKE(i915))
 555                 kbl_ctx_workarounds_init(engine);
 556         else if (IS_GEMINILAKE(i915))
 557                 glk_ctx_workarounds_init(engine);
 558         else if (IS_COFFEELAKE(i915))
 559                 cfl_ctx_workarounds_init(engine);
 560         else if (IS_CANNONLAKE(i915))
 561                 cnl_ctx_workarounds_init(engine);
 562         else if (IS_ICELAKE(i915))
 563                 icl_ctx_workarounds_init(engine);
 564         else
 565                 MISSING_CASE(INTEL_GEN(i915));
 566
 567         wa_init_finish(wal);
 568 }
 569
 570 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 571 {
 572         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 573         struct i915_wa *wa;
 574         unsigned int i;
 575         u32 *cs;
 576         int ret;
 577
 578         if (wal->count == 0)
 579                 return 0;
 580
 581         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 582         if (ret)
 583                 return ret;
 584
 585         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 586         if (IS_ERR(cs))
 587                 return PTR_ERR(cs);
 588
 589         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 590         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 591                 *cs++ = i915_mmio_reg_offset(wa->reg);
 592                 *cs++ = wa->val;
 593         }
 594         *cs++ = MI_NOOP;
 595
 596         intel_ring_advance(rq, cs);
 597
 598         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 599         if (ret)
 600                 return ret;
 601
 602         return 0;
 603 }
 604
 605 static void
 606 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 607 {
 608         struct i915_wa wa = {
 609                 .reg = reg,
 610                 .mask = val,
 611                 .val = _MASKED_BIT_ENABLE(val)
 612         };
 613
 614         _wa_add(wal, &wa);
 615 }
 616
 617 static void
 618 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask,
 619                    u32 val)
 620 {
 621         struct i915_wa wa = {
 622                 .reg = reg,
 623                 .mask = mask,
 624                 .val = val
 625         };
 626
 627         _wa_add(wal, &wa);
 628 }
 629
 630 static void
 631 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 632 {
 633         wa_write_masked_or(wal, reg, ~0, val);
 634 }
 635
 636 static void
 637 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 638 {
 639         wa_write_masked_or(wal, reg, val, val);
 640 }
 641
 642 static void
 643 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 644 {
 645         /* WaDisableKillLogic:bxt,skl,kbl */
 646         if (!IS_COFFEELAKE(i915))
 647                 wa_write_or(wal,
 648                             GAM_ECOCHK,
 649                             ECOCHK_DIS_TLB);
 650
 651         if (HAS_LLC(i915)) {
 652                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 653                  *
 654                  * Must match Display Engine. See
 655                  * WaCompressedResourceDisplayNewHashMode.
 656                  */
 657                 wa_write_or(wal,
 658                             MMCD_MISC_CTRL,
 659                             MMCD_PCLA | MMCD_HOTSPOT_EN);
 660         }
 661
 662         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
 663         wa_write_or(wal,
 664                     GAM_ECOCHK,
 665                     BDW_DISABLE_HDC_INVALIDATION);
 666 }
 667
 668 static void
 669 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 670 {
 671         gen9_gt_workarounds_init(i915, wal);
 672
 673         /* WaDisableGafsUnitClkGating:skl */
 674         wa_write_or(wal,
 675                     GEN7_UCGCTL4,
 676                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 677
 678         /* WaInPlaceDecompressionHang:skl */
 679         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
 680                 wa_write_or(wal,
 681                             GEN9_GAMT_ECO_REG_RW_IA,
 682                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 683 }
 684
 685 static void
 686 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 687 {
 688         gen9_gt_workarounds_init(i915, wal);
 689
 690         /* WaInPlaceDecompressionHang:bxt */
 691         wa_write_or(wal,
 692                     GEN9_GAMT_ECO_REG_RW_IA,
 693                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 694 }
 695
 696 static void
 697 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 698 {
 699         gen9_gt_workarounds_init(i915, wal);
 700
 701         /* WaDisableDynamicCreditSharing:kbl */
 702         if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
 703                 wa_write_or(wal,
 704                             GAMT_CHKN_BIT_REG,
 705                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
 706
 707         /* WaDisableGafsUnitClkGating:kbl */
 708         wa_write_or(wal,
 709                     GEN7_UCGCTL4,
 710                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 711
 712         /* WaInPlaceDecompressionHang:kbl */
 713         wa_write_or(wal,
 714                     GEN9_GAMT_ECO_REG_RW_IA,
 715                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 716 }
 717
 718 static void
 719 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 720 {
 721         gen9_gt_workarounds_init(i915, wal);
 722 }
 723
 724 static void
 725 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 726 {
 727         gen9_gt_workarounds_init(i915, wal);
 728
 729         /* WaDisableGafsUnitClkGating:cfl */
 730         wa_write_or(wal,
 731                     GEN7_UCGCTL4,
 732                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 733
 734         /* WaInPlaceDecompressionHang:cfl */
 735         wa_write_or(wal,
 736                     GEN9_GAMT_ECO_REG_RW_IA,
 737                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 738 }
 739
 740 static void
 741 wa_init_mcr(struct drm_i915_private *dev_priv, struct i915_wa_list *wal)
 742 {
 743         const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
 744         u32 mcr_slice_subslice_mask;
 745
 746         /*
 747          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
 748          * L3Banks could be fused off in single slice scenario. If that is
 749          * the case, we might need to program MCR select to a valid L3Bank
 750          * by default, to make sure we correctly read certain registers
 751          * later on (in the range 0xB100 - 0xB3FF).
 752          * This might be incompatible with
 753          * WaProgramMgsrForCorrectSliceSpecificMmioReads.
 754          * Fortunately, this should not happen in production hardware, so
 755          * we only assert that this is the case (instead of implementing
 756          * something more complex that requires checking the range of every
 757          * MMIO read).
 758          */
 759         if (INTEL_GEN(dev_priv) >= 10 &&
 760             is_power_of_2(sseu->slice_mask)) {
 761                 /*
 762                  * read FUSE3 for enabled L3 Bank IDs, if L3 Bank matches
 763                  * enabled subslice, no need to redirect MCR packet
 764                  */
 765                 u32 slice = fls(sseu->slice_mask);
 766                 u32 fuse3 = I915_READ(GEN10_MIRROR_FUSE3);
 767                 u8 ss_mask = sseu->subslice_mask[slice];
 768
 769                 u8 enabled_mask = (ss_mask | ss_mask >>
 770                                    GEN10_L3BANK_PAIR_COUNT) & GEN10_L3BANK_MASK;
 771                 u8 disabled_mask = fuse3 & GEN10_L3BANK_MASK;
 772
 773                 /*
 774                  * Production silicon should have matched L3Bank and
 775                  * subslice enabled
 776                  */
 777                 WARN_ON((enabled_mask & disabled_mask) != enabled_mask);
 778         }
 779
 780         if (INTEL_GEN(dev_priv) >= 11)
 781                 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
 782                                           GEN11_MCR_SUBSLICE_MASK;
 783         else
 784                 mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
 785                                           GEN8_MCR_SUBSLICE_MASK;
 786         /*
 787          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
 788          * Before any MMIO read into slice/subslice specific registers, MCR
 789          * packet control register needs to be programmed to point to any
 790          * enabled s/ss pair. Otherwise, incorrect values will be returned.
 791          * This means each subsequent MMIO read will be forwarded to an
 792          * specific s/ss combination, but this is OK since these registers
 793          * are consistent across s/ss in almost all cases. In the rare
 794          * occasions, such as INSTDONE, where this value is dependent
 795          * on s/ss combo, the read should be done with read_subslice_reg.
 796          */
 797         wa_write_masked_or(wal,
 798                            GEN8_MCR_SELECTOR,
 799                            mcr_slice_subslice_mask,
 800                            intel_calculate_mcr_s_ss_select(dev_priv));
 801 }
 802
 803 static void
 804 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 805 {
 806         wa_init_mcr(i915, wal);
 807
 808         /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
 809         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 810                 wa_write_or(wal,
 811                             GAMT_CHKN_BIT_REG,
 812                             GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
 813
 814         /* WaInPlaceDecompressionHang:cnl */
 815         wa_write_or(wal,
 816                     GEN9_GAMT_ECO_REG_RW_IA,
 817                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 818 }
 819
 820 static void
 821 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 822 {
 823         wa_init_mcr(i915, wal);
 824
 825         /* WaInPlaceDecompressionHang:icl */
 826         wa_write_or(wal,
 827                     GEN9_GAMT_ECO_REG_RW_IA,
 828                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 829
 830         /* WaModifyGamTlbPartitioning:icl */
 831         wa_write_masked_or(wal,
 832                            GEN11_GACB_PERF_CTRL,
 833                            GEN11_HASH_CTRL_MASK,
 834                            GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
 835
 836         /* Wa_1405766107:icl
 837          * Formerly known as WaCL2SFHalfMaxAlloc
 838          */
 839         wa_write_or(wal,
 840                     GEN11_LSN_UNSLCVC,
 841                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
 842                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
 843
 844         /* Wa_220166154:icl
 845          * Formerly known as WaDisCtxReload
 846          */
 847         wa_write_or(wal,
 848                     GEN8_GAMW_ECO_DEV_RW_IA,
 849                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
 850
 851         /* Wa_1405779004:icl (pre-prod) */
 852         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 853                 wa_write_or(wal,
 854                             SLICE_UNIT_LEVEL_CLKGATE,
 855                             MSCUNIT_CLKGATE_DIS);
 856
 857         /* Wa_1406680159:icl */
 858         wa_write_or(wal,
 859                     SUBSLICE_UNIT_LEVEL_CLKGATE,
 860                     GWUNIT_CLKGATE_DIS);
 861
 862         /* Wa_1406838659:icl (pre-prod) */
 863         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 864                 wa_write_or(wal,
 865                             INF_UNIT_LEVEL_CLKGATE,
 866                             CGPSF_CLKGATE_DIS);
 867
 868         /* Wa_1406463099:icl
 869          * Formerly known as WaGamTlbPendError
 870          */
 871         wa_write_or(wal,
 872                     GAMT_CHKN_BIT_REG,
 873                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
 874 }
 875
 876 static void
 877 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
 878 {
 879         if (INTEL_GEN(i915) < 8)
 880                 return;
 881         else if (IS_BROADWELL(i915))
 882                 return;
 883         else if (IS_CHERRYVIEW(i915))
 884                 return;
 885         else if (IS_SKYLAKE(i915))
 886                 skl_gt_workarounds_init(i915, wal);
 887         else if (IS_BROXTON(i915))
 888                 bxt_gt_workarounds_init(i915, wal);
 889         else if (IS_KABYLAKE(i915))
 890                 kbl_gt_workarounds_init(i915, wal);
 891         else if (IS_GEMINILAKE(i915))
 892                 glk_gt_workarounds_init(i915, wal);
 893         else if (IS_COFFEELAKE(i915))
 894                 cfl_gt_workarounds_init(i915, wal);
 895         else if (IS_CANNONLAKE(i915))
 896                 cnl_gt_workarounds_init(i915, wal);
 897         else if (IS_ICELAKE(i915))
 898                 icl_gt_workarounds_init(i915, wal);
 899         else
 900                 MISSING_CASE(INTEL_GEN(i915));
 901 }
 902
 903 void intel_gt_init_workarounds(struct drm_i915_private *i915)
 904 {
 905         struct i915_wa_list *wal = &i915->gt_wa_list;
 906
 907         wa_init_start(wal, "GT");
 908         gt_init_workarounds(i915, wal);
 909         wa_init_finish(wal);
 910 }
 911
 912 static enum forcewake_domains
 913 wal_get_fw_for_rmw(struct drm_i915_private *dev_priv,
 914                    const struct i915_wa_list *wal)
 915 {
 916         enum forcewake_domains fw = 0;
 917         struct i915_wa *wa;
 918         unsigned int i;
 919
 920         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 921                 fw |= intel_uncore_forcewake_for_reg(dev_priv,
 922                                                      wa->reg,
 923                                                      FW_REG_READ |
 924                                                      FW_REG_WRITE);
 925
 926         return fw;
 927 }
 928
 929 static void
 930 wa_list_apply(struct drm_i915_private *dev_priv, const struct i915_wa_list *wal)
 931 {
 932         enum forcewake_domains fw;
 933         unsigned long flags;
 934         struct i915_wa *wa;
 935         unsigned int i;
 936
 937         if (!wal->count)
 938                 return;
 939
 940         fw = wal_get_fw_for_rmw(dev_priv, wal);
 941
 942         spin_lock_irqsave(&dev_priv->uncore.lock, flags);
 943         intel_uncore_forcewake_get__locked(dev_priv, fw);
 944
 945         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 946                 u32 val = I915_READ_FW(wa->reg);
 947
 948                 val &= ~wa->mask;
 949                 val |= wa->val;
 950
 951                 I915_WRITE_FW(wa->reg, val);
 952         }
 953
 954         intel_uncore_forcewake_put__locked(dev_priv, fw);
 955         spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
 956 }
 957
 958 void intel_gt_apply_workarounds(struct drm_i915_private *dev_priv)
 959 {
 960         wa_list_apply(dev_priv, &dev_priv->gt_wa_list);
 961 }
 962
 963 static bool
 964 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
 965 {
 966         if ((cur ^ wa->val) & wa->mask) {
 967                 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x, mask=%x)\n",
 968                           name, from, i915_mmio_reg_offset(wa->reg), cur,
 969                           cur & wa->mask, wa->val, wa->mask);
 970
 971                 return false;
 972         }
 973
 974         return true;
 975 }
 976
 977 static bool wa_list_verify(struct drm_i915_private *dev_priv,
 978                            const struct i915_wa_list *wal,
 979                            const char *from)
 980 {
 981         struct i915_wa *wa;
 982         unsigned int i;
 983         bool ok = true;
 984
 985         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 986                 ok &= wa_verify(wa, I915_READ(wa->reg), wal->name, from);
 987
 988         return ok;
 989 }
 990
 991 bool intel_gt_verify_workarounds(struct drm_i915_private *dev_priv,
 992                                  const char *from)
 993 {
 994         return wa_list_verify(dev_priv, &dev_priv->gt_wa_list, from);
 995 }
 996
 997 static void
 998 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
 999 {
1000         struct i915_wa wa = {
1001                 .reg = reg
1002         };
1003
1004         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1005                 return;
1006
1007         _wa_add(wal, &wa);
1008 }
1009
1010 static void gen9_whitelist_build(struct i915_wa_list *w)
1011 {
1012         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1013         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1014
1015         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1016         whitelist_reg(w, GEN8_CS_CHICKEN1);
1017
1018         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1019         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1020 }
1021
1022 static void skl_whitelist_build(struct i915_wa_list *w)
1023 {
1024         gen9_whitelist_build(w);
1025
1026         /* WaDisableLSQCROPERFforOCL:skl */
1027         whitelist_reg(w, GEN8_L3SQCREG4);
1028 }
1029
1030 static void bxt_whitelist_build(struct i915_wa_list *w)
1031 {
1032         gen9_whitelist_build(w);
1033 }
1034
1035 static void kbl_whitelist_build(struct i915_wa_list *w)
1036 {
1037         gen9_whitelist_build(w);
1038
1039         /* WaDisableLSQCROPERFforOCL:kbl */
1040         whitelist_reg(w, GEN8_L3SQCREG4);
1041 }
1042
1043 static void glk_whitelist_build(struct i915_wa_list *w)
1044 {
1045         gen9_whitelist_build(w);
1046
1047         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1048         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1049 }
1050
1051 static void cfl_whitelist_build(struct i915_wa_list *w)
1052 {
1053         gen9_whitelist_build(w);
1054 }
1055
1056 static void cnl_whitelist_build(struct i915_wa_list *w)
1057 {
1058         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1059         whitelist_reg(w, GEN8_CS_CHICKEN1);
1060 }
1061
1062 static void icl_whitelist_build(struct i915_wa_list *w)
1063 {
1064         /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1065         whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1066
1067         /* WaAllowUMDToModifySamplerMode:icl */
1068         whitelist_reg(w, GEN10_SAMPLER_MODE);
1069 }
1070
1071 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1072 {
1073         struct drm_i915_private *i915 = engine->i915;
1074         struct i915_wa_list *w = &engine->whitelist;
1075
1076         GEM_BUG_ON(engine->id != RCS);
1077
1078         wa_init_start(w, "whitelist");
1079
1080         if (INTEL_GEN(i915) < 8)
1081                 return;
1082         else if (IS_BROADWELL(i915))
1083                 return;
1084         else if (IS_CHERRYVIEW(i915))
1085                 return;
1086         else if (IS_SKYLAKE(i915))
1087                 skl_whitelist_build(w);
1088         else if (IS_BROXTON(i915))
1089                 bxt_whitelist_build(w);
1090         else if (IS_KABYLAKE(i915))
1091                 kbl_whitelist_build(w);
1092         else if (IS_GEMINILAKE(i915))
1093                 glk_whitelist_build(w);
1094         else if (IS_COFFEELAKE(i915))
1095                 cfl_whitelist_build(w);
1096         else if (IS_CANNONLAKE(i915))
1097                 cnl_whitelist_build(w);
1098         else if (IS_ICELAKE(i915))
1099                 icl_whitelist_build(w);
1100         else
1101                 MISSING_CASE(INTEL_GEN(i915));
1102
1103         wa_init_finish(w);
1104 }
1105
1106 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1107 {
1108         struct drm_i915_private *dev_priv = engine->i915;
1109         const struct i915_wa_list *wal = &engine->whitelist;
1110         const u32 base = engine->mmio_base;
1111         struct i915_wa *wa;
1112         unsigned int i;
1113
1114         if (!wal->count)
1115                 return;
1116
1117         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1118                 I915_WRITE(RING_FORCE_TO_NONPRIV(base, i),
1119                            i915_mmio_reg_offset(wa->reg));
1120
1121         /* And clear the rest just in case of garbage */
1122         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1123                 I915_WRITE(RING_FORCE_TO_NONPRIV(base, i),
1124                            i915_mmio_reg_offset(RING_NOPID(base)));
1125 }
1126
1127 static void
1128 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1129 {
1130         struct drm_i915_private *i915 = engine->i915;
1131
1132         if (IS_ICELAKE(i915)) {
1133                 /* This is not an Wa. Enable for better image quality */
1134                 wa_masked_en(wal,
1135                              _3D_CHICKEN3,
1136                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1137
1138                 /* WaPipelineFlushCoherentLines:icl */
1139                 wa_write_or(wal,
1140                             GEN8_L3SQCREG4,
1141                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1142
1143                 /*
1144                  * Wa_1405543622:icl
1145                  * Formerly known as WaGAPZPriorityScheme
1146                  */
1147                 wa_write_or(wal,
1148                             GEN8_GARBCNTL,
1149                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1150
1151                 /*
1152                  * Wa_1604223664:icl
1153                  * Formerly known as WaL3BankAddressHashing
1154                  */
1155                 wa_write_masked_or(wal,
1156                                    GEN8_GARBCNTL,
1157                                    GEN11_HASH_CTRL_EXCL_MASK,
1158                                    GEN11_HASH_CTRL_EXCL_BIT0);
1159                 wa_write_masked_or(wal,
1160                                    GEN11_GLBLINVL,
1161                                    GEN11_BANK_HASH_ADDR_EXCL_MASK,
1162                                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1163
1164                 /*
1165                  * Wa_1405733216:icl
1166                  * Formerly known as WaDisableCleanEvicts
1167                  */
1168                 wa_write_or(wal,
1169                             GEN8_L3SQCREG4,
1170                             GEN11_LQSC_CLEAN_EVICT_DISABLE);
1171
1172                 /* WaForwardProgressSoftReset:icl */
1173                 wa_write_or(wal,
1174                             GEN10_SCRATCH_LNCF2,
1175                             PMFLUSHDONE_LNICRSDROP |
1176                             PMFLUSH_GAPL3UNBLOCK |
1177                             PMFLUSHDONE_LNEBLK);
1178
1179                 /* Wa_1406609255:icl (pre-prod) */
1180                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1181                         wa_write_or(wal,
1182                                     GEN7_SARCHKMD,
1183                                     GEN7_DISABLE_DEMAND_PREFETCH |
1184                                     GEN7_DISABLE_SAMPLER_PREFETCH);
1185         }
1186
1187         if (IS_GEN(i915, 9) || IS_CANNONLAKE(i915)) {
1188                 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,cnl */
1189                 wa_masked_en(wal,
1190                              GEN7_FF_SLICE_CS_CHICKEN1,
1191                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1192         }
1193
1194         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1195                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1196                 wa_write_or(wal,
1197                             GEN8_GARBCNTL,
1198                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1199         }
1200
1201         if (IS_BROXTON(i915)) {
1202                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1203                 wa_masked_en(wal,
1204                              FF_SLICE_CS_CHICKEN2,
1205                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1206         }
1207
1208         if (IS_GEN(i915, 9)) {
1209                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1210                 wa_masked_en(wal,
1211                              GEN9_CSFE_CHICKEN1_RCS,
1212                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1213
1214                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1215                 wa_write_or(wal,
1216                             BDW_SCRATCH1,
1217                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1218
1219                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1220                 if (IS_GEN9_LP(i915))
1221                         wa_write_masked_or(wal,
1222                                            GEN8_L3SQCREG1,
1223                                            L3_PRIO_CREDITS_MASK,
1224                                            L3_GENERAL_PRIO_CREDITS(62) |
1225                                            L3_HIGH_PRIO_CREDITS(2));
1226
1227                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1228                 wa_write_or(wal,
1229                             GEN8_L3SQCREG4,
1230                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1231         }
1232 }
1233
1234 static void
1235 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1236 {
1237         struct drm_i915_private *i915 = engine->i915;
1238
1239         /* WaKBLVECSSemaphoreWaitPoll:kbl */
1240         if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1241                 wa_write(wal,
1242                          RING_SEMA_WAIT_POLL(engine->mmio_base),
1243                          1);
1244         }
1245 }
1246
1247 static void
1248 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1249 {
1250         if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 8))
1251                 return;
1252
1253         if (engine->id == RCS)
1254                 rcs_engine_wa_init(engine, wal);
1255         else
1256                 xcs_engine_wa_init(engine, wal);
1257 }
1258
1259 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1260 {
1261         struct i915_wa_list *wal = &engine->wa_list;
1262
1263         if (GEM_WARN_ON(INTEL_GEN(engine->i915) < 8))
1264                 return;
1265
1266         wa_init_start(wal, engine->name);
1267         engine_init_workarounds(engine, wal);
1268         wa_init_finish(wal);
1269 }
1270
1271 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1272 {
1273         wa_list_apply(engine->i915, &engine->wa_list);
1274 }
1275
1276 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1277 #include "selftests/intel_workarounds.c"
1278 #endif