drivers/gpu/drm/i915/gt/intel_workarounds.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2018 Intel Corporation
   5  */
   6
   7 #include "i915_drv.h"
   8 #include "intel_workarounds.h"
   9
  10 /**
  11  * DOC: Hardware workarounds
  12  *
  13  * This file is intended as a central place to implement most [1]_ of the
  14  * required workarounds for hardware to work as originally intended. They fall
  15  * in five basic categories depending on how/when they are applied:
  16  *
  17  * - Workarounds that touch registers that are saved/restored to/from the HW
  18  *   context image. The list is emitted (via Load Register Immediate commands)
  19  *   everytime a new context is created.
  20  * - GT workarounds. The list of these WAs is applied whenever these registers
  21  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
  22  * - Display workarounds. The list is applied during display clock-gating
  23  *   initialization.
  24  * - Workarounds that whitelist a privileged register, so that UMDs can manage
  25  *   them directly. This is just a special case of a MMMIO workaround (as we
  26  *   write the list of these to/be-whitelisted registers to some special HW
  27  *   registers).
  28  * - Workaround batchbuffers, that get executed automatically by the hardware
  29  *   on every HW context restore.
  30  *
  31  * .. [1] Please notice that there are other WAs that, due to their nature,
  32  *    cannot be applied from a central place. Those are peppered around the rest
  33  *    of the code, as needed.
  34  *
  35  * .. [2] Technically, some registers are powercontext saved & restored, so they
  36  *    survive a suspend/resume. In practice, writing them again is not too
  37  *    costly and simplifies things. We can revisit this in the future.
  38  *
  39  * Layout
  40  * ''''''
  41  *
  42  * Keep things in this file ordered by WA type, as per the above (context, GT,
  43  * display, register whitelist, batchbuffer). Then, inside each type, keep the
  44  * following order:
  45  *
  46  * - Infrastructure functions and macros
  47  * - WAs per platform in standard gen/chrono order
  48  * - Public functions to init or apply the given workaround type.
  49  */
  50
  51 static void wa_init_start(struct i915_wa_list *wal, const char *name)
  52 {
  53         wal->name = name;
  54 }
  55
  56 #define WA_LIST_CHUNK (1 << 4)
  57
  58 static void wa_init_finish(struct i915_wa_list *wal)
  59 {
  60         /* Trim unused entries. */
  61         if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) {
  62                 struct i915_wa *list = kmemdup(wal->list,
  63                                                wal->count * sizeof(*list),
  64                                                GFP_KERNEL);
  65
  66                 if (list) {
  67                         kfree(wal->list);
  68                         wal->list = list;
  69                 }
  70         }
  71
  72         if (!wal->count)
  73                 return;
  74
  75         DRM_DEBUG_DRIVER("Initialized %u %s workarounds\n",
  76                          wal->wa_count, wal->name);
  77 }
  78
  79 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa)
  80 {
  81         unsigned int addr = i915_mmio_reg_offset(wa->reg);
  82         unsigned int start = 0, end = wal->count;
  83         const unsigned int grow = WA_LIST_CHUNK;
  84         struct i915_wa *wa_;
  85
  86         GEM_BUG_ON(!is_power_of_2(grow));
  87
  88         if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */
  89                 struct i915_wa *list;
  90
  91                 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa),
  92                                      GFP_KERNEL);
  93                 if (!list) {
  94                         DRM_ERROR("No space for workaround init!\n");
  95                         return;
  96                 }
  97
  98                 if (wal->list)
  99                         memcpy(list, wal->list, sizeof(*wa) * wal->count);
 100
 101                 wal->list = list;
 102         }
 103
 104         while (start < end) {
 105                 unsigned int mid = start + (end - start) / 2;
 106
 107                 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) {
 108                         start = mid + 1;
 109                 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) {
 110                         end = mid;
 111                 } else {
 112                         wa_ = &wal->list[mid];
 113
 114                         if ((wa->mask & ~wa_->mask) == 0) {
 115                                 DRM_ERROR("Discarding overwritten w/a for reg %04x (mask: %08x, value: %08x)\n",
 116                                           i915_mmio_reg_offset(wa_->reg),
 117                                           wa_->mask, wa_->val);
 118
 119                                 wa_->val &= ~wa->mask;
 120                         }
 121
 122                         wal->wa_count++;
 123                         wa_->val |= wa->val;
 124                         wa_->mask |= wa->mask;
 125                         wa_->read |= wa->read;
 126                         return;
 127                 }
 128         }
 129
 130         wal->wa_count++;
 131         wa_ = &wal->list[wal->count++];
 132         *wa_ = *wa;
 133
 134         while (wa_-- > wal->list) {
 135                 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) ==
 136                            i915_mmio_reg_offset(wa_[1].reg));
 137                 if (i915_mmio_reg_offset(wa_[1].reg) >
 138                     i915_mmio_reg_offset(wa_[0].reg))
 139                         break;
 140
 141                 swap(wa_[1], wa_[0]);
 142         }
 143 }
 144
 145 static void
 146 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask,
 147                    u32 val)
 148 {
 149         struct i915_wa wa = {
 150                 .reg  = reg,
 151                 .mask = mask,
 152                 .val  = val,
 153                 .read = mask,
 154         };
 155
 156         _wa_add(wal, &wa);
 157 }
 158
 159 static void
 160 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 161 {
 162         wa_write_masked_or(wal, reg, val, _MASKED_BIT_ENABLE(val));
 163 }
 164
 165 static void
 166 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 167 {
 168         wa_write_masked_or(wal, reg, ~0, val);
 169 }
 170
 171 static void
 172 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 173 {
 174         wa_write_masked_or(wal, reg, val, val);
 175 }
 176
 177 static void
 178 ignore_wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 mask, u32 val)
 179 {
 180         struct i915_wa wa = {
 181                 .reg  = reg,
 182                 .mask = mask,
 183                 .val  = val,
 184                 /* Bonkers HW, skip verifying */
 185         };
 186
 187         _wa_add(wal, &wa);
 188 }
 189
 190 #define WA_SET_BIT_MASKED(addr, mask) \
 191         wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_ENABLE(mask))
 192
 193 #define WA_CLR_BIT_MASKED(addr, mask) \
 194         wa_write_masked_or(wal, (addr), (mask), _MASKED_BIT_DISABLE(mask))
 195
 196 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 197         wa_write_masked_or(wal, (addr), (mask), _MASKED_FIELD((mask), (value)))
 198
 199 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine,
 200                                       struct i915_wa_list *wal)
 201 {
 202         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 203
 204         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 205         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 206
 207         /* WaDisablePartialInstShootdown:bdw,chv */
 208         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 209                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 210
 211         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 212          * workaround for for a possible hang in the unlikely event a TLB
 213          * invalidation occurs during a PSD flush.
 214          */
 215         /* WaForceEnableNonCoherent:bdw,chv */
 216         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 217         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 218                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 219                           HDC_FORCE_NON_COHERENT);
 220
 221         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 222          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 223          *  polygons in the same 8x4 pixel/sample area to be processed without
 224          *  stalling waiting for the earlier ones to write to Hierarchical Z
 225          *  buffer."
 226          *
 227          * This optimization is off by default for BDW and CHV; turn it on.
 228          */
 229         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 230
 231         /* Wa4x4STCOptimizationDisable:bdw,chv */
 232         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 233
 234         /*
 235          * BSpec recommends 8x4 when MSAA is used,
 236          * however in practice 16x4 seems fastest.
 237          *
 238          * Note that PS/WM thread counts depend on the WIZ hashing
 239          * disable bit, which we don't touch here, but it's good
 240          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 241          */
 242         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 243                             GEN6_WIZ_HASHING_MASK,
 244                             GEN6_WIZ_HASHING_16x4);
 245 }
 246
 247 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine,
 248                                      struct i915_wa_list *wal)
 249 {
 250         struct drm_i915_private *i915 = engine->i915;
 251
 252         gen8_ctx_workarounds_init(engine, wal);
 253
 254         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 255         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 256
 257         /* WaDisableDopClockGating:bdw
 258          *
 259          * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
 260          * to disable EUTC clock gating.
 261          */
 262         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 263                           DOP_CLOCK_GATING_DISABLE);
 264
 265         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 266                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 267
 268         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 269                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 270                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 271                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 272                           (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 273 }
 274
 275 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine,
 276                                      struct i915_wa_list *wal)
 277 {
 278         gen8_ctx_workarounds_init(engine, wal);
 279
 280         /* WaDisableThreadStallDopClockGating:chv */
 281         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 282
 283         /* Improve HiZ throughput on CHV. */
 284         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 285 }
 286
 287 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine,
 288                                       struct i915_wa_list *wal)
 289 {
 290         struct drm_i915_private *i915 = engine->i915;
 291
 292         if (HAS_LLC(i915)) {
 293                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 294                  *
 295                  * Must match Display Engine. See
 296                  * WaCompressedResourceDisplayNewHashMode.
 297                  */
 298                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 299                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
 300                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 301                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
 302         }
 303
 304         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
 305         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
 306         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 307                           FLOW_CONTROL_ENABLE |
 308                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 309
 310         /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
 311         if (!IS_COFFEELAKE(i915))
 312                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 313                                   GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 314
 315         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 316         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 317         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 318                           GEN9_ENABLE_YV12_BUGFIX |
 319                           GEN9_ENABLE_GPGPU_PREEMPTION);
 320
 321         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
 322         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
 323         WA_SET_BIT_MASKED(CACHE_MODE_1,
 324                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 325                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
 326
 327         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
 328         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 329                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 330
 331         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 332         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 333                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 334                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 335
 336         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 337          * both tied to WaForceContextSaveRestoreNonCoherent
 338          * in some hsds for skl. We keep the tie for all gen9. The
 339          * documentation is a bit hazy and so we want to get common behaviour,
 340          * even though there is no clear evidence we would need both on kbl/bxt.
 341          * This area has been source of system hangs so we play it safe
 342          * and mimic the skl regardless of what bspec says.
 343          *
 344          * Use Force Non-Coherent whenever executing a 3D context. This
 345          * is a workaround for a possible hang in the unlikely event
 346          * a TLB invalidation occurs during a PSD flush.
 347          */
 348
 349         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
 350         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 351                           HDC_FORCE_NON_COHERENT);
 352
 353         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 354         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915))
 355                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 356                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 357
 358         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 359         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 360
 361         /*
 362          * Supporting preemption with fine-granularity requires changes in the
 363          * batch buffer programming. Since we can't break old userspace, we
 364          * need to set our default preemption level to safe value. Userspace is
 365          * still able to use more fine-grained preemption levels, since in
 366          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
 367          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
 368          * not real HW workarounds, but merely a way to start using preemption
 369          * while maintaining old contract with userspace.
 370          */
 371
 372         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
 373         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 374
 375         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
 376         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 377                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 378                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 379
 380         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
 381         if (IS_GEN9_LP(i915))
 382                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
 383 }
 384
 385 static void skl_tune_iz_hashing(struct intel_engine_cs *engine,
 386                                 struct i915_wa_list *wal)
 387 {
 388         struct drm_i915_private *i915 = engine->i915;
 389         u8 vals[3] = { 0, 0, 0 };
 390         unsigned int i;
 391
 392         for (i = 0; i < 3; i++) {
 393                 u8 ss;
 394
 395                 /*
 396                  * Only consider slices where one, and only one, subslice has 7
 397                  * EUs
 398                  */
 399                 if (!is_power_of_2(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]))
 400                         continue;
 401
 402                 /*
 403                  * subslice_7eu[i] != 0 (because of the check above) and
 404                  * ss_max == 4 (maximum number of subslices possible per slice)
 405                  *
 406                  * ->    0 <= ss <= 3;
 407                  */
 408                 ss = ffs(RUNTIME_INFO(i915)->sseu.subslice_7eu[i]) - 1;
 409                 vals[i] = 3 - ss;
 410         }
 411
 412         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 413                 return;
 414
 415         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 416         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 417                             GEN9_IZ_HASHING_MASK(2) |
 418                             GEN9_IZ_HASHING_MASK(1) |
 419                             GEN9_IZ_HASHING_MASK(0),
 420                             GEN9_IZ_HASHING(2, vals[2]) |
 421                             GEN9_IZ_HASHING(1, vals[1]) |
 422                             GEN9_IZ_HASHING(0, vals[0]));
 423 }
 424
 425 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine,
 426                                      struct i915_wa_list *wal)
 427 {
 428         gen9_ctx_workarounds_init(engine, wal);
 429         skl_tune_iz_hashing(engine, wal);
 430 }
 431
 432 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine,
 433                                      struct i915_wa_list *wal)
 434 {
 435         gen9_ctx_workarounds_init(engine, wal);
 436
 437         /* WaDisableThreadStallDopClockGating:bxt */
 438         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 439                           STALL_DOP_GATING_DISABLE);
 440
 441         /* WaToEnableHwFixForPushConstHWBug:bxt */
 442         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 443                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 444 }
 445
 446 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine,
 447                                      struct i915_wa_list *wal)
 448 {
 449         struct drm_i915_private *i915 = engine->i915;
 450
 451         gen9_ctx_workarounds_init(engine, wal);
 452
 453         /* WaToEnableHwFixForPushConstHWBug:kbl */
 454         if (IS_KBL_REVID(i915, KBL_REVID_C0, REVID_FOREVER))
 455                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 456                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 457
 458         /* WaDisableSbeCacheDispatchPortSharing:kbl */
 459         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 460                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 461 }
 462
 463 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine,
 464                                      struct i915_wa_list *wal)
 465 {
 466         gen9_ctx_workarounds_init(engine, wal);
 467
 468         /* WaToEnableHwFixForPushConstHWBug:glk */
 469         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 470                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 471 }
 472
 473 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine,
 474                                      struct i915_wa_list *wal)
 475 {
 476         gen9_ctx_workarounds_init(engine, wal);
 477
 478         /* WaToEnableHwFixForPushConstHWBug:cfl */
 479         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 480                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 481
 482         /* WaDisableSbeCacheDispatchPortSharing:cfl */
 483         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
 484                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 485 }
 486
 487 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine,
 488                                      struct i915_wa_list *wal)
 489 {
 490         struct drm_i915_private *i915 = engine->i915;
 491
 492         /* WaForceContextSaveRestoreNonCoherent:cnl */
 493         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
 494                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
 495
 496         /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
 497         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 498                 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
 499
 500         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
 501         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 502                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 503
 504         /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
 505         if (IS_CNL_REVID(i915, 0, CNL_REVID_B0))
 506                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
 507                                   GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
 508
 509         /* WaPushConstantDereferenceHoldDisable:cnl */
 510         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
 511
 512         /* FtrEnableFastAnisoL1BankingFix:cnl */
 513         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
 514
 515         /* WaDisable3DMidCmdPreemption:cnl */
 516         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
 517
 518         /* WaDisableGPGPUMidCmdPreemption:cnl */
 519         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 520                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 521                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
 522
 523         /* WaDisableEarlyEOT:cnl */
 524         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
 525 }
 526
 527 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 528                                      struct i915_wa_list *wal)
 529 {
 530         struct drm_i915_private *i915 = engine->i915;
 531
 532         /* WaDisableBankHangMode:icl */
 533         wa_write(wal,
 534                  GEN8_L3CNTLREG,
 535                  intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) |
 536                  GEN8_ERRDETBCTRL);
 537
 538         /* Wa_1604370585:icl (pre-prod)
 539          * Formerly known as WaPushConstantDereferenceHoldDisable
 540          */
 541         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 542                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 543                                   PUSH_CONSTANT_DEREF_DISABLE);
 544
 545         /* WaForceEnableNonCoherent:icl
 546          * This is not the same workaround as in early Gen9 platforms, where
 547          * lacking this could cause system hangs, but coherency performance
 548          * overhead is high and only a few compute workloads really need it
 549          * (the register is whitelisted in hardware now, so UMDs can opt in
 550          * for coherency if they have a good reason).
 551          */
 552         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
 553
 554         /* Wa_2006611047:icl (pre-prod)
 555          * Formerly known as WaDisableImprovedTdlClkGating
 556          */
 557         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 558                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 559                                   GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 560
 561         /* Wa_2006665173:icl (pre-prod) */
 562         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 563                 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
 564                                   GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC);
 565
 566         /* WaEnableFloatBlendOptimization:icl */
 567         wa_write_masked_or(wal,
 568                            GEN10_CACHE_MODE_SS,
 569                            0, /* write-only, so skip validation */
 570                            _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
 571
 572         /* WaDisableGPGPUMidThreadPreemption:icl */
 573         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
 574                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
 575                             GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 576
 577         /* allow headerless messages for preemptible GPGPU context */
 578         WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE,
 579                           GEN11_SAMPLER_ENABLE_HEADLESS_MSG);
 580 }
 581
 582 static void
 583 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine,
 584                            struct i915_wa_list *wal,
 585                            const char *name)
 586 {
 587         struct drm_i915_private *i915 = engine->i915;
 588
 589         if (engine->class != RENDER_CLASS)
 590                 return;
 591
 592         wa_init_start(wal, name);
 593
 594         if (IS_GEN(i915, 11))
 595                 icl_ctx_workarounds_init(engine, wal);
 596         else if (IS_CANNONLAKE(i915))
 597                 cnl_ctx_workarounds_init(engine, wal);
 598         else if (IS_COFFEELAKE(i915))
 599                 cfl_ctx_workarounds_init(engine, wal);
 600         else if (IS_GEMINILAKE(i915))
 601                 glk_ctx_workarounds_init(engine, wal);
 602         else if (IS_KABYLAKE(i915))
 603                 kbl_ctx_workarounds_init(engine, wal);
 604         else if (IS_BROXTON(i915))
 605                 bxt_ctx_workarounds_init(engine, wal);
 606         else if (IS_SKYLAKE(i915))
 607                 skl_ctx_workarounds_init(engine, wal);
 608         else if (IS_CHERRYVIEW(i915))
 609                 chv_ctx_workarounds_init(engine, wal);
 610         else if (IS_BROADWELL(i915))
 611                 bdw_ctx_workarounds_init(engine, wal);
 612         else if (INTEL_GEN(i915) < 8)
 613                 return;
 614         else
 615                 MISSING_CASE(INTEL_GEN(i915));
 616
 617         wa_init_finish(wal);
 618 }
 619
 620 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 621 {
 622         __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context");
 623 }
 624
 625 int intel_engine_emit_ctx_wa(struct i915_request *rq)
 626 {
 627         struct i915_wa_list *wal = &rq->engine->ctx_wa_list;
 628         struct i915_wa *wa;
 629         unsigned int i;
 630         u32 *cs;
 631         int ret;
 632
 633         if (wal->count == 0)
 634                 return 0;
 635
 636         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 637         if (ret)
 638                 return ret;
 639
 640         cs = intel_ring_begin(rq, (wal->count * 2 + 2));
 641         if (IS_ERR(cs))
 642                 return PTR_ERR(cs);
 643
 644         *cs++ = MI_LOAD_REGISTER_IMM(wal->count);
 645         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 646                 *cs++ = i915_mmio_reg_offset(wa->reg);
 647                 *cs++ = wa->val;
 648         }
 649         *cs++ = MI_NOOP;
 650
 651         intel_ring_advance(rq, cs);
 652
 653         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
 654         if (ret)
 655                 return ret;
 656
 657         return 0;
 658 }
 659
 660 static void
 661 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 662 {
 663         /* WaDisableKillLogic:bxt,skl,kbl */
 664         if (!IS_COFFEELAKE(i915))
 665                 wa_write_or(wal,
 666                             GAM_ECOCHK,
 667                             ECOCHK_DIS_TLB);
 668
 669         if (HAS_LLC(i915)) {
 670                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
 671                  *
 672                  * Must match Display Engine. See
 673                  * WaCompressedResourceDisplayNewHashMode.
 674                  */
 675                 wa_write_or(wal,
 676                             MMCD_MISC_CTRL,
 677                             MMCD_PCLA | MMCD_HOTSPOT_EN);
 678         }
 679
 680         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
 681         wa_write_or(wal,
 682                     GAM_ECOCHK,
 683                     BDW_DISABLE_HDC_INVALIDATION);
 684 }
 685
 686 static void
 687 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 688 {
 689         gen9_gt_workarounds_init(i915, wal);
 690
 691         /* WaDisableGafsUnitClkGating:skl */
 692         wa_write_or(wal,
 693                     GEN7_UCGCTL4,
 694                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 695
 696         /* WaInPlaceDecompressionHang:skl */
 697         if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER))
 698                 wa_write_or(wal,
 699                             GEN9_GAMT_ECO_REG_RW_IA,
 700                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 701 }
 702
 703 static void
 704 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 705 {
 706         gen9_gt_workarounds_init(i915, wal);
 707
 708         /* WaInPlaceDecompressionHang:bxt */
 709         wa_write_or(wal,
 710                     GEN9_GAMT_ECO_REG_RW_IA,
 711                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 712 }
 713
 714 static void
 715 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 716 {
 717         gen9_gt_workarounds_init(i915, wal);
 718
 719         /* WaDisableDynamicCreditSharing:kbl */
 720         if (IS_KBL_REVID(i915, 0, KBL_REVID_B0))
 721                 wa_write_or(wal,
 722                             GAMT_CHKN_BIT_REG,
 723                             GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
 724
 725         /* WaDisableGafsUnitClkGating:kbl */
 726         wa_write_or(wal,
 727                     GEN7_UCGCTL4,
 728                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 729
 730         /* WaInPlaceDecompressionHang:kbl */
 731         wa_write_or(wal,
 732                     GEN9_GAMT_ECO_REG_RW_IA,
 733                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 734 }
 735
 736 static void
 737 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 738 {
 739         gen9_gt_workarounds_init(i915, wal);
 740 }
 741
 742 static void
 743 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 744 {
 745         gen9_gt_workarounds_init(i915, wal);
 746
 747         /* WaDisableGafsUnitClkGating:cfl */
 748         wa_write_or(wal,
 749                     GEN7_UCGCTL4,
 750                     GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 751
 752         /* WaInPlaceDecompressionHang:cfl */
 753         wa_write_or(wal,
 754                     GEN9_GAMT_ECO_REG_RW_IA,
 755                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 756 }
 757
 758 static void
 759 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
 760 {
 761         const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu;
 762         u32 mcr_slice_subslice_mask;
 763
 764         /*
 765          * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl
 766          * L3Banks could be fused off in single slice scenario. If that is
 767          * the case, we might need to program MCR select to a valid L3Bank
 768          * by default, to make sure we correctly read certain registers
 769          * later on (in the range 0xB100 - 0xB3FF).
 770          * This might be incompatible with
 771          * WaProgramMgsrForCorrectSliceSpecificMmioReads.
 772          * Fortunately, this should not happen in production hardware, so
 773          * we only assert that this is the case (instead of implementing
 774          * something more complex that requires checking the range of every
 775          * MMIO read).
 776          */
 777         if (INTEL_GEN(i915) >= 10 &&
 778             is_power_of_2(sseu->slice_mask)) {
 779                 /*
 780                  * read FUSE3 for enabled L3 Bank IDs, if L3 Bank matches
 781                  * enabled subslice, no need to redirect MCR packet
 782                  */
 783                 u32 slice = fls(sseu->slice_mask);
 784                 u32 fuse3 =
 785                         intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3);
 786                 u8 ss_mask = sseu->subslice_mask[slice];
 787
 788                 u8 enabled_mask = (ss_mask | ss_mask >>
 789                                    GEN10_L3BANK_PAIR_COUNT) & GEN10_L3BANK_MASK;
 790                 u8 disabled_mask = fuse3 & GEN10_L3BANK_MASK;
 791
 792                 /*
 793                  * Production silicon should have matched L3Bank and
 794                  * subslice enabled
 795                  */
 796                 WARN_ON((enabled_mask & disabled_mask) != enabled_mask);
 797         }
 798
 799         if (INTEL_GEN(i915) >= 11)
 800                 mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
 801                                           GEN11_MCR_SUBSLICE_MASK;
 802         else
 803                 mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
 804                                           GEN8_MCR_SUBSLICE_MASK;
 805         /*
 806          * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl
 807          * Before any MMIO read into slice/subslice specific registers, MCR
 808          * packet control register needs to be programmed to point to any
 809          * enabled s/ss pair. Otherwise, incorrect values will be returned.
 810          * This means each subsequent MMIO read will be forwarded to an
 811          * specific s/ss combination, but this is OK since these registers
 812          * are consistent across s/ss in almost all cases. In the rare
 813          * occasions, such as INSTDONE, where this value is dependent
 814          * on s/ss combo, the read should be done with read_subslice_reg.
 815          */
 816         wa_write_masked_or(wal,
 817                            GEN8_MCR_SELECTOR,
 818                            mcr_slice_subslice_mask,
 819                            intel_calculate_mcr_s_ss_select(i915));
 820 }
 821
 822 static void
 823 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 824 {
 825         wa_init_mcr(i915, wal);
 826
 827         /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
 828         if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0))
 829                 wa_write_or(wal,
 830                             GAMT_CHKN_BIT_REG,
 831                             GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
 832
 833         /* WaInPlaceDecompressionHang:cnl */
 834         wa_write_or(wal,
 835                     GEN9_GAMT_ECO_REG_RW_IA,
 836                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 837 }
 838
 839 static void
 840 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 841 {
 842         wa_init_mcr(i915, wal);
 843
 844         /* WaInPlaceDecompressionHang:icl */
 845         wa_write_or(wal,
 846                     GEN9_GAMT_ECO_REG_RW_IA,
 847                     GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 848
 849         /* WaModifyGamTlbPartitioning:icl */
 850         wa_write_masked_or(wal,
 851                            GEN11_GACB_PERF_CTRL,
 852                            GEN11_HASH_CTRL_MASK,
 853                            GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
 854
 855         /* Wa_1405766107:icl
 856          * Formerly known as WaCL2SFHalfMaxAlloc
 857          */
 858         wa_write_or(wal,
 859                     GEN11_LSN_UNSLCVC,
 860                     GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
 861                     GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
 862
 863         /* Wa_220166154:icl
 864          * Formerly known as WaDisCtxReload
 865          */
 866         wa_write_or(wal,
 867                     GEN8_GAMW_ECO_DEV_RW_IA,
 868                     GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
 869
 870         /* Wa_1405779004:icl (pre-prod) */
 871         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 872                 wa_write_or(wal,
 873                             SLICE_UNIT_LEVEL_CLKGATE,
 874                             MSCUNIT_CLKGATE_DIS);
 875
 876         /* Wa_1406680159:icl */
 877         wa_write_or(wal,
 878                     SUBSLICE_UNIT_LEVEL_CLKGATE,
 879                     GWUNIT_CLKGATE_DIS);
 880
 881         /* Wa_1406838659:icl (pre-prod) */
 882         if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
 883                 wa_write_or(wal,
 884                             INF_UNIT_LEVEL_CLKGATE,
 885                             CGPSF_CLKGATE_DIS);
 886
 887         /* Wa_1406463099:icl
 888          * Formerly known as WaGamTlbPendError
 889          */
 890         wa_write_or(wal,
 891                     GAMT_CHKN_BIT_REG,
 892                     GAMT_CHKN_DISABLE_L3_COH_PIPE);
 893 }
 894
 895 static void
 896 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
 897 {
 898         if (IS_GEN(i915, 11))
 899                 icl_gt_workarounds_init(i915, wal);
 900         else if (IS_CANNONLAKE(i915))
 901                 cnl_gt_workarounds_init(i915, wal);
 902         else if (IS_COFFEELAKE(i915))
 903                 cfl_gt_workarounds_init(i915, wal);
 904         else if (IS_GEMINILAKE(i915))
 905                 glk_gt_workarounds_init(i915, wal);
 906         else if (IS_KABYLAKE(i915))
 907                 kbl_gt_workarounds_init(i915, wal);
 908         else if (IS_BROXTON(i915))
 909                 bxt_gt_workarounds_init(i915, wal);
 910         else if (IS_SKYLAKE(i915))
 911                 skl_gt_workarounds_init(i915, wal);
 912         else if (INTEL_GEN(i915) <= 8)
 913                 return;
 914         else
 915                 MISSING_CASE(INTEL_GEN(i915));
 916 }
 917
 918 void intel_gt_init_workarounds(struct drm_i915_private *i915)
 919 {
 920         struct i915_wa_list *wal = &i915->gt_wa_list;
 921
 922         wa_init_start(wal, "GT");
 923         gt_init_workarounds(i915, wal);
 924         wa_init_finish(wal);
 925 }
 926
 927 static enum forcewake_domains
 928 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
 929 {
 930         enum forcewake_domains fw = 0;
 931         struct i915_wa *wa;
 932         unsigned int i;
 933
 934         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
 935                 fw |= intel_uncore_forcewake_for_reg(uncore,
 936                                                      wa->reg,
 937                                                      FW_REG_READ |
 938                                                      FW_REG_WRITE);
 939
 940         return fw;
 941 }
 942
 943 static bool
 944 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
 945 {
 946         if ((cur ^ wa->val) & wa->read) {
 947                 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x, mask=%x)\n",
 948                           name, from, i915_mmio_reg_offset(wa->reg),
 949                           cur, cur & wa->read,
 950                           wa->val, wa->mask);
 951
 952                 return false;
 953         }
 954
 955         return true;
 956 }
 957
 958 static void
 959 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
 960 {
 961         enum forcewake_domains fw;
 962         unsigned long flags;
 963         struct i915_wa *wa;
 964         unsigned int i;
 965
 966         if (!wal->count)
 967                 return;
 968
 969         fw = wal_get_fw_for_rmw(uncore, wal);
 970
 971         spin_lock_irqsave(&uncore->lock, flags);
 972         intel_uncore_forcewake_get__locked(uncore, fw);
 973
 974         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
 975                 intel_uncore_rmw_fw(uncore, wa->reg, wa->mask, wa->val);
 976                 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 977                         wa_verify(wa,
 978                                   intel_uncore_read_fw(uncore, wa->reg),
 979                                   wal->name, "application");
 980         }
 981
 982         intel_uncore_forcewake_put__locked(uncore, fw);
 983         spin_unlock_irqrestore(&uncore->lock, flags);
 984 }
 985
 986 void intel_gt_apply_workarounds(struct drm_i915_private *i915)
 987 {
 988         wa_list_apply(&i915->uncore, &i915->gt_wa_list);
 989 }
 990
 991 static bool wa_list_verify(struct intel_uncore *uncore,
 992                            const struct i915_wa_list *wal,
 993                            const char *from)
 994 {
 995         struct i915_wa *wa;
 996         unsigned int i;
 997         bool ok = true;
 998
 999         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1000                 ok &= wa_verify(wa,
1001                                 intel_uncore_read(uncore, wa->reg),
1002                                 wal->name, from);
1003
1004         return ok;
1005 }
1006
1007 bool intel_gt_verify_workarounds(struct drm_i915_private *i915,
1008                                  const char *from)
1009 {
1010         return wa_list_verify(&i915->uncore, &i915->gt_wa_list, from);
1011 }
1012
1013 static void
1014 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg)
1015 {
1016         struct i915_wa wa = {
1017                 .reg = reg
1018         };
1019
1020         if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS))
1021                 return;
1022
1023         _wa_add(wal, &wa);
1024 }
1025
1026 static void gen9_whitelist_build(struct i915_wa_list *w)
1027 {
1028         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
1029         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
1030
1031         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1032         whitelist_reg(w, GEN8_CS_CHICKEN1);
1033
1034         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
1035         whitelist_reg(w, GEN8_HDC_CHICKEN1);
1036 }
1037
1038 static void skl_whitelist_build(struct i915_wa_list *w)
1039 {
1040         gen9_whitelist_build(w);
1041
1042         /* WaDisableLSQCROPERFforOCL:skl */
1043         whitelist_reg(w, GEN8_L3SQCREG4);
1044 }
1045
1046 static void bxt_whitelist_build(struct i915_wa_list *w)
1047 {
1048         gen9_whitelist_build(w);
1049 }
1050
1051 static void kbl_whitelist_build(struct i915_wa_list *w)
1052 {
1053         gen9_whitelist_build(w);
1054
1055         /* WaDisableLSQCROPERFforOCL:kbl */
1056         whitelist_reg(w, GEN8_L3SQCREG4);
1057 }
1058
1059 static void glk_whitelist_build(struct i915_wa_list *w)
1060 {
1061         gen9_whitelist_build(w);
1062
1063         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
1064         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1065 }
1066
1067 static void cfl_whitelist_build(struct i915_wa_list *w)
1068 {
1069         gen9_whitelist_build(w);
1070 }
1071
1072 static void cnl_whitelist_build(struct i915_wa_list *w)
1073 {
1074         /* WaEnablePreemptionGranularityControlByUMD:cnl */
1075         whitelist_reg(w, GEN8_CS_CHICKEN1);
1076 }
1077
1078 static void icl_whitelist_build(struct i915_wa_list *w)
1079 {
1080         /* WaAllowUMDToModifyHalfSliceChicken7:icl */
1081         whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7);
1082
1083         /* WaAllowUMDToModifySamplerMode:icl */
1084         whitelist_reg(w, GEN10_SAMPLER_MODE);
1085
1086         /* WaEnableStateCacheRedirectToCS:icl */
1087         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
1088 }
1089
1090 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
1091 {
1092         struct drm_i915_private *i915 = engine->i915;
1093         struct i915_wa_list *w = &engine->whitelist;
1094
1095         if (engine->class != RENDER_CLASS)
1096                 return;
1097
1098         wa_init_start(w, "whitelist");
1099
1100         if (IS_GEN(i915, 11))
1101                 icl_whitelist_build(w);
1102         else if (IS_CANNONLAKE(i915))
1103                 cnl_whitelist_build(w);
1104         else if (IS_COFFEELAKE(i915))
1105                 cfl_whitelist_build(w);
1106         else if (IS_GEMINILAKE(i915))
1107                 glk_whitelist_build(w);
1108         else if (IS_KABYLAKE(i915))
1109                 kbl_whitelist_build(w);
1110         else if (IS_BROXTON(i915))
1111                 bxt_whitelist_build(w);
1112         else if (IS_SKYLAKE(i915))
1113                 skl_whitelist_build(w);
1114         else if (INTEL_GEN(i915) <= 8)
1115                 return;
1116         else
1117                 MISSING_CASE(INTEL_GEN(i915));
1118
1119         wa_init_finish(w);
1120 }
1121
1122 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
1123 {
1124         const struct i915_wa_list *wal = &engine->whitelist;
1125         struct intel_uncore *uncore = engine->uncore;
1126         const u32 base = engine->mmio_base;
1127         struct i915_wa *wa;
1128         unsigned int i;
1129
1130         if (!wal->count)
1131                 return;
1132
1133         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1134                 intel_uncore_write(uncore,
1135                                    RING_FORCE_TO_NONPRIV(base, i),
1136                                    i915_mmio_reg_offset(wa->reg));
1137
1138         /* And clear the rest just in case of garbage */
1139         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
1140                 intel_uncore_write(uncore,
1141                                    RING_FORCE_TO_NONPRIV(base, i),
1142                                    i915_mmio_reg_offset(RING_NOPID(base)));
1143 }
1144
1145 static void
1146 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1147 {
1148         struct drm_i915_private *i915 = engine->i915;
1149
1150         if (IS_GEN(i915, 11)) {
1151                 /* This is not an Wa. Enable for better image quality */
1152                 wa_masked_en(wal,
1153                              _3D_CHICKEN3,
1154                              _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE);
1155
1156                 /* WaPipelineFlushCoherentLines:icl */
1157                 ignore_wa_write_or(wal,
1158                                    GEN8_L3SQCREG4,
1159                                    GEN8_LQSC_FLUSH_COHERENT_LINES,
1160                                    GEN8_LQSC_FLUSH_COHERENT_LINES);
1161
1162                 /*
1163                  * Wa_1405543622:icl
1164                  * Formerly known as WaGAPZPriorityScheme
1165                  */
1166                 wa_write_or(wal,
1167                             GEN8_GARBCNTL,
1168                             GEN11_ARBITRATION_PRIO_ORDER_MASK);
1169
1170                 /*
1171                  * Wa_1604223664:icl
1172                  * Formerly known as WaL3BankAddressHashing
1173                  */
1174                 wa_write_masked_or(wal,
1175                                    GEN8_GARBCNTL,
1176                                    GEN11_HASH_CTRL_EXCL_MASK,
1177                                    GEN11_HASH_CTRL_EXCL_BIT0);
1178                 wa_write_masked_or(wal,
1179                                    GEN11_GLBLINVL,
1180                                    GEN11_BANK_HASH_ADDR_EXCL_MASK,
1181                                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
1182
1183                 /*
1184                  * Wa_1405733216:icl
1185                  * Formerly known as WaDisableCleanEvicts
1186                  */
1187                 ignore_wa_write_or(wal,
1188                                    GEN8_L3SQCREG4,
1189                                    GEN11_LQSC_CLEAN_EVICT_DISABLE,
1190                                    GEN11_LQSC_CLEAN_EVICT_DISABLE);
1191
1192                 /* WaForwardProgressSoftReset:icl */
1193                 wa_write_or(wal,
1194                             GEN10_SCRATCH_LNCF2,
1195                             PMFLUSHDONE_LNICRSDROP |
1196                             PMFLUSH_GAPL3UNBLOCK |
1197                             PMFLUSHDONE_LNEBLK);
1198
1199                 /* Wa_1406609255:icl (pre-prod) */
1200                 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0))
1201                         wa_write_or(wal,
1202                                     GEN7_SARCHKMD,
1203                                     GEN7_DISABLE_DEMAND_PREFETCH |
1204                                     GEN7_DISABLE_SAMPLER_PREFETCH);
1205         }
1206
1207         if (IS_GEN_RANGE(i915, 9, 11)) {
1208                 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl */
1209                 wa_masked_en(wal,
1210                              GEN7_FF_SLICE_CS_CHICKEN1,
1211                              GEN9_FFSC_PERCTX_PREEMPT_CTRL);
1212         }
1213
1214         if (IS_SKYLAKE(i915) || IS_KABYLAKE(i915) || IS_COFFEELAKE(i915)) {
1215                 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */
1216                 wa_write_or(wal,
1217                             GEN8_GARBCNTL,
1218                             GEN9_GAPS_TSV_CREDIT_DISABLE);
1219         }
1220
1221         if (IS_BROXTON(i915)) {
1222                 /* WaDisablePooledEuLoadBalancingFix:bxt */
1223                 wa_masked_en(wal,
1224                              FF_SLICE_CS_CHICKEN2,
1225                              GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1226         }
1227
1228         if (IS_GEN(i915, 9)) {
1229                 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
1230                 wa_masked_en(wal,
1231                              GEN9_CSFE_CHICKEN1_RCS,
1232                              GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE);
1233
1234                 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
1235                 wa_write_or(wal,
1236                             BDW_SCRATCH1,
1237                             GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
1238
1239                 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
1240                 if (IS_GEN9_LP(i915))
1241                         wa_write_masked_or(wal,
1242                                            GEN8_L3SQCREG1,
1243                                            L3_PRIO_CREDITS_MASK,
1244                                            L3_GENERAL_PRIO_CREDITS(62) |
1245                                            L3_HIGH_PRIO_CREDITS(2));
1246
1247                 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
1248                 wa_write_or(wal,
1249                             GEN8_L3SQCREG4,
1250                             GEN8_LQSC_FLUSH_COHERENT_LINES);
1251         }
1252 }
1253
1254 static void
1255 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1256 {
1257         struct drm_i915_private *i915 = engine->i915;
1258
1259         /* WaKBLVECSSemaphoreWaitPoll:kbl */
1260         if (IS_KBL_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) {
1261                 wa_write(wal,
1262                          RING_SEMA_WAIT_POLL(engine->mmio_base),
1263                          1);
1264         }
1265 }
1266
1267 static void
1268 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal)
1269 {
1270         if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 8))
1271                 return;
1272
1273         if (engine->id == RCS0)
1274                 rcs_engine_wa_init(engine, wal);
1275         else
1276                 xcs_engine_wa_init(engine, wal);
1277 }
1278
1279 void intel_engine_init_workarounds(struct intel_engine_cs *engine)
1280 {
1281         struct i915_wa_list *wal = &engine->wa_list;
1282
1283         if (GEM_WARN_ON(INTEL_GEN(engine->i915) < 8))
1284                 return;
1285
1286         wa_init_start(wal, engine->name);
1287         engine_init_workarounds(engine, wal);
1288         wa_init_finish(wal);
1289 }
1290
1291 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
1292 {
1293         wa_list_apply(engine->uncore, &engine->wa_list);
1294 }
1295
1296 static struct i915_vma *
1297 create_scratch(struct i915_address_space *vm, int count)
1298 {
1299         struct drm_i915_gem_object *obj;
1300         struct i915_vma *vma;
1301         unsigned int size;
1302         int err;
1303
1304         size = round_up(count * sizeof(u32), PAGE_SIZE);
1305         obj = i915_gem_object_create_internal(vm->i915, size);
1306         if (IS_ERR(obj))
1307                 return ERR_CAST(obj);
1308
1309         i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
1310
1311         vma = i915_vma_instance(obj, vm, NULL);
1312         if (IS_ERR(vma)) {
1313                 err = PTR_ERR(vma);
1314                 goto err_obj;
1315         }
1316
1317         err = i915_vma_pin(vma, 0, 0,
1318                            i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
1319         if (err)
1320                 goto err_obj;
1321
1322         return vma;
1323
1324 err_obj:
1325         i915_gem_object_put(obj);
1326         return ERR_PTR(err);
1327 }
1328
1329 static int
1330 wa_list_srm(struct i915_request *rq,
1331             const struct i915_wa_list *wal,
1332             struct i915_vma *vma)
1333 {
1334         const struct i915_wa *wa;
1335         unsigned int i;
1336         u32 srm, *cs;
1337
1338         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1339         if (INTEL_GEN(rq->i915) >= 8)
1340                 srm++;
1341
1342         cs = intel_ring_begin(rq, 4 * wal->count);
1343         if (IS_ERR(cs))
1344                 return PTR_ERR(cs);
1345
1346         for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
1347                 *cs++ = srm;
1348                 *cs++ = i915_mmio_reg_offset(wa->reg);
1349                 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
1350                 *cs++ = 0;
1351         }
1352         intel_ring_advance(rq, cs);
1353
1354         return 0;
1355 }
1356
1357 static int engine_wa_list_verify(struct intel_context *ce,
1358                                  const struct i915_wa_list * const wal,
1359                                  const char *from)
1360 {
1361         const struct i915_wa *wa;
1362         struct i915_request *rq;
1363         struct i915_vma *vma;
1364         unsigned int i;
1365         u32 *results;
1366         int err;
1367
1368         if (!wal->count)
1369                 return 0;
1370
1371         vma = create_scratch(&ce->engine->i915->ggtt.vm, wal->count);
1372         if (IS_ERR(vma))
1373                 return PTR_ERR(vma);
1374
1375         rq = intel_context_create_request(ce);
1376         if (IS_ERR(rq)) {
1377                 err = PTR_ERR(rq);
1378                 goto err_vma;
1379         }
1380
1381         err = wa_list_srm(rq, wal, vma);
1382         if (err)
1383                 goto err_vma;
1384
1385         i915_request_add(rq);
1386         if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) {
1387                 err = -ETIME;
1388                 goto err_vma;
1389         }
1390
1391         results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
1392         if (IS_ERR(results)) {
1393                 err = PTR_ERR(results);
1394                 goto err_vma;
1395         }
1396
1397         err = 0;
1398         for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
1399                 if (!wa_verify(wa, results[i], wal->name, from))
1400                         err = -ENXIO;
1401
1402         i915_gem_object_unpin_map(vma->obj);
1403
1404 err_vma:
1405         i915_vma_unpin(vma);
1406         i915_vma_put(vma);
1407         return err;
1408 }
1409
1410 int intel_engine_verify_workarounds(struct intel_engine_cs *engine,
1411                                     const char *from)
1412 {
1413         return engine_wa_list_verify(engine->kernel_context,
1414                                      &engine->wa_list,
1415                                      from);
1416 }
1417
1418 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1419 #include "selftest_workarounds.c"
1420 #endif