drivers/gpu/drm/i915/intel_clock_gating.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display.h"
  30 #include "display/intel_display_trace.h"
  31 #include "display/skl_watermark.h"
  32
  33 #include "gt/intel_engine_regs.h"
  34 #include "gt/intel_gt.h"
  35 #include "gt/intel_gt_mcr.h"
  36 #include "gt/intel_gt_regs.h"
  37
  38 #include "i915_drv.h"
  39 #include "i915_reg.h"
  40 #include "intel_clock_gating.h"
  41 #include "intel_mchbar_regs.h"
  42 #include "vlv_sideband.h"
  43
  44 struct drm_i915_clock_gating_funcs {
  45         void (*init_clock_gating)(struct drm_i915_private *i915);
  46 };
  47
  48 static void gen9_init_clock_gating(struct drm_i915_private *i915)
  49 {
  50         if (HAS_LLC(i915)) {
  51                 /*
  52                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  53                  * Display WA #0390: skl,kbl
  54                  *
  55                  * Must match Sampler, Pixel Back End, and Media. See
  56                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  57                  */
  58                 intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
  59         }
  60
  61         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  62         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
  63
  64         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  65         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
  66
  67         /*
  68          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  69          * Display WA #0859: skl,bxt,kbl,glk,cfl
  70          */
  71         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
  72 }
  73
  74 static void bxt_init_clock_gating(struct drm_i915_private *i915)
  75 {
  76         gen9_init_clock_gating(i915);
  77
  78         /* WaDisableSDEUnitClockGating:bxt */
  79         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  80
  81         /*
  82          * FIXME:
  83          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  84          */
  85         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  86
  87         /*
  88          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  89          * to stay fully on.
  90          */
  91         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
  92                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
  93                            PWM1_GATING_DIS | PWM2_GATING_DIS);
  94
  95         /*
  96          * Lower the display internal timeout.
  97          * This is needed to avoid any hard hangs when DSI port PLL
  98          * is off and a MMIO access is attempted by any privilege
  99          * application, using batch buffers or any other means.
 100          */
 101         intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 102
 103         /*
 104          * WaFbcTurnOffFbcWatermark:bxt
 105          * Display WA #0562: bxt
 106          */
 107         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 108
 109         /*
 110          * WaFbcHighMemBwCorruptionAvoidance:bxt
 111          * Display WA #0883: bxt
 112          */
 113         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 114 }
 115
 116 static void glk_init_clock_gating(struct drm_i915_private *i915)
 117 {
 118         gen9_init_clock_gating(i915);
 119
 120         /*
 121          * WaDisablePWMClockGating:glk
 122          * Backlight PWM may stop in the asserted state, causing backlight
 123          * to stay fully on.
 124          */
 125         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
 126                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
 127                            PWM1_GATING_DIS | PWM2_GATING_DIS);
 128 }
 129
 130 static void ibx_init_clock_gating(struct drm_i915_private *i915)
 131 {
 132         /*
 133          * On Ibex Peak and Cougar Point, we need to disable clock
 134          * gating for the panel power sequencer or it will fail to
 135          * start up when no ports are active.
 136          */
 137         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
 138 }
 139
 140 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
 141 {
 142         enum pipe pipe;
 143
 144         for_each_pipe(dev_priv, pipe) {
 145                 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
 146
 147                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
 148                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
 149         }
 150 }
 151
 152 static void ilk_init_clock_gating(struct drm_i915_private *i915)
 153 {
 154         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 155
 156         /*
 157          * Required for FBC
 158          * WaFbcDisableDpfcClockGating:ilk
 159          */
 160         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
 161                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
 162                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
 163
 164         intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
 165                            MARIUNIT_CLOCK_GATE_DISABLE |
 166                            SVSMUNIT_CLOCK_GATE_DISABLE);
 167         intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
 168                            VFMUNIT_CLOCK_GATE_DISABLE);
 169
 170         /*
 171          * According to the spec the following bits should be set in
 172          * order to enable memory self-refresh
 173          * The bit 22/21 of 0x42004
 174          * The bit 5 of 0x42020
 175          * The bit 15 of 0x45000
 176          */
 177         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 178                            (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 179                             ILK_DPARB_GATE | ILK_VSDPFD_FULL));
 180         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
 181         intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
 182                            (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
 183                             DISP_FBC_WM_DIS));
 184
 185         /*
 186          * Based on the document from hardware guys the following bits
 187          * should be set unconditionally in order to enable FBC.
 188          * The bit 22 of 0x42000
 189          * The bit 22 of 0x42004
 190          * The bit 7,8,9 of 0x42020.
 191          */
 192         if (IS_IRONLAKE_M(i915)) {
 193                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
 194                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 195                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
 196         }
 197
 198         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 199
 200         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 201
 202         g4x_disable_trickle_feed(i915);
 203
 204         ibx_init_clock_gating(i915);
 205 }
 206
 207 static void cpt_init_clock_gating(struct drm_i915_private *i915)
 208 {
 209         enum pipe pipe;
 210         u32 val;
 211
 212         /*
 213          * On Ibex Peak and Cougar Point, we need to disable clock
 214          * gating for the panel power sequencer or it will fail to
 215          * start up when no ports are active.
 216          */
 217         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
 218                            PCH_DPLUNIT_CLOCK_GATE_DISABLE |
 219                            PCH_CPUNIT_CLOCK_GATE_DISABLE);
 220         intel_uncore_rmw(&i915->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
 221         /* The below fixes the weird display corruption, a few pixels shifted
 222          * downward, on (only) LVDS of some HP laptops with IVY.
 223          */
 224         for_each_pipe(i915, pipe) {
 225                 val = intel_uncore_read(&i915->uncore, TRANS_CHICKEN2(pipe));
 226                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 227                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 228                 if (i915->display.vbt.fdi_rx_polarity_inverted)
 229                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 230                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
 231                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
 232                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN2(pipe), val);
 233         }
 234         /* WADP0ClockGatingDisable */
 235         for_each_pipe(i915, pipe) {
 236                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN1(pipe),
 237                                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 238         }
 239 }
 240
 241 static void gen6_check_mch_setup(struct drm_i915_private *i915)
 242 {
 243         u32 tmp;
 244
 245         tmp = intel_uncore_read(&i915->uncore, MCH_SSKPD);
 246         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
 247                 drm_dbg_kms(&i915->drm,
 248                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
 249                             tmp);
 250 }
 251
 252 static void gen6_init_clock_gating(struct drm_i915_private *i915)
 253 {
 254         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 255
 256         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 257
 258         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 259
 260         intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
 261                            intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
 262                            GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
 263                            GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 264
 265         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
 266          * gating disable must be set.  Failure to set it results in
 267          * flickering pixels due to Z write ordering failures after
 268          * some amount of runtime in the Mesa "fire" demo, and Unigine
 269          * Sanctuary and Tropics, and apparently anything else with
 270          * alpha test or pixel discard.
 271          *
 272          * According to the spec, bit 11 (RCCUNIT) must also be set,
 273          * but we didn't debug actual testcases to find it out.
 274          *
 275          * WaDisableRCCUnitClockGating:snb
 276          * WaDisableRCPBUnitClockGating:snb
 277          */
 278         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 279                            GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
 280                            GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 281
 282         /*
 283          * According to the spec the following bits should be
 284          * set in order to enable memory self-refresh and fbc:
 285          * The bit21 and bit22 of 0x42000
 286          * The bit21 and bit22 of 0x42004
 287          * The bit5 and bit7 of 0x42020
 288          * The bit14 of 0x70180
 289          * The bit14 of 0x71180
 290          *
 291          * WaFbcAsynchFlipDisableFbcQueue:snb
 292          */
 293         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
 294                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
 295                            ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
 296         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 297                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 298                            ILK_DPARB_GATE | ILK_VSDPFD_FULL);
 299         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
 300                            intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
 301                            ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
 302                            ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 303
 304         g4x_disable_trickle_feed(i915);
 305
 306         cpt_init_clock_gating(i915);
 307
 308         gen6_check_mch_setup(i915);
 309 }
 310
 311 static void lpt_init_clock_gating(struct drm_i915_private *i915)
 312 {
 313         /*
 314          * TODO: this bit should only be enabled when really needed, then
 315          * disabled when not needed anymore in order to save power.
 316          */
 317         if (HAS_PCH_LPT_LP(i915))
 318                 intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D,
 319                                  0, PCH_LP_PARTITION_LEVEL_DISABLE);
 320
 321         /* WADPOClockGatingDisable:hsw */
 322         intel_uncore_rmw(&i915->uncore, TRANS_CHICKEN1(PIPE_A),
 323                          0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 324 }
 325
 326 static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
 327                                    int general_prio_credits,
 328                                    int high_prio_credits)
 329 {
 330         u32 misccpctl;
 331         u32 val;
 332
 333         /* WaTempDisableDOPClkGating:bdw */
 334         misccpctl = intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 335                                      GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 336
 337         val = intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 338         val &= ~L3_PRIO_CREDITS_MASK;
 339         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
 340         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
 341         intel_gt_mcr_multicast_write(to_gt(i915), GEN8_L3SQCREG1, val);
 342
 343         /*
 344          * Wait at least 100 clocks before re-enabling clock gating.
 345          * See the definition of L3SQCREG1 in BSpec.
 346          */
 347         intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 348         udelay(1);
 349         intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
 350 }
 351
 352 static void icl_init_clock_gating(struct drm_i915_private *i915)
 353 {
 354         /* Wa_1409120013:icl,ehl */
 355         intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 356                            DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 357
 358         /*Wa_14010594013:icl, ehl */
 359         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1,
 360                          0, ICL_DELAY_PMRSP);
 361 }
 362
 363 static void gen12lp_init_clock_gating(struct drm_i915_private *i915)
 364 {
 365         /* Wa_1409120013 */
 366         if (DISPLAY_VER(i915) == 12)
 367                 intel_uncore_write(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 368                                    DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 369
 370         /* Wa_14013723622:tgl,rkl,dg1,adl-s */
 371         if (DISPLAY_VER(i915) == 12)
 372                 intel_uncore_rmw(&i915->uncore, CLKREQ_POLICY,
 373                                  CLKREQ_POLICY_MEM_UP_OVRD, 0);
 374 }
 375
 376 static void adlp_init_clock_gating(struct drm_i915_private *i915)
 377 {
 378         gen12lp_init_clock_gating(i915);
 379
 380         /* Wa_22011091694:adlp */
 381         intel_de_rmw(i915, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
 382
 383         /* Bspec/49189 Initialize Sequence */
 384         intel_de_rmw(i915, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
 385 }
 386
 387 static void xehpsdv_init_clock_gating(struct drm_i915_private *i915)
 388 {
 389         /* Wa_22010146351:xehpsdv */
 390         if (IS_XEHPSDV_GRAPHICS_STEP(i915, STEP_A0, STEP_B0))
 391                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 392 }
 393
 394 static void dg2_init_clock_gating(struct drm_i915_private *i915)
 395 {
 396         /* Wa_22010954014:dg2 */
 397         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
 398                          SGSI_SIDECLK_DIS);
 399 }
 400
 401 static void pvc_init_clock_gating(struct drm_i915_private *i915)
 402 {
 403         /* Wa_14012385139:pvc */
 404         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 405                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
 406
 407         /* Wa_22010954014:pvc */
 408         if (IS_PVC_BD_STEP(i915, STEP_A0, STEP_B0))
 409                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
 410 }
 411
 412 static void cnp_init_clock_gating(struct drm_i915_private *i915)
 413 {
 414         if (!HAS_PCH_CNP(i915))
 415                 return;
 416
 417         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
 418         intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
 419 }
 420
 421 static void cfl_init_clock_gating(struct drm_i915_private *i915)
 422 {
 423         cnp_init_clock_gating(i915);
 424         gen9_init_clock_gating(i915);
 425
 426         /* WAC6entrylatency:cfl */
 427         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 428
 429         /*
 430          * WaFbcTurnOffFbcWatermark:cfl
 431          * Display WA #0562: cfl
 432          */
 433         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 434
 435         /*
 436          * WaFbcNukeOnHostModify:cfl
 437          * Display WA #0873: cfl
 438          */
 439         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 440                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 441 }
 442
 443 static void kbl_init_clock_gating(struct drm_i915_private *i915)
 444 {
 445         gen9_init_clock_gating(i915);
 446
 447         /* WAC6entrylatency:kbl */
 448         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 449
 450         /* WaDisableSDEUnitClockGating:kbl */
 451         if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
 452                 intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6,
 453                                  0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 454
 455         /* WaDisableGamClockGating:kbl */
 456         if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
 457                 intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
 458                                  0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 459
 460         /*
 461          * WaFbcTurnOffFbcWatermark:kbl
 462          * Display WA #0562: kbl
 463          */
 464         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 465
 466         /*
 467          * WaFbcNukeOnHostModify:kbl
 468          * Display WA #0873: kbl
 469          */
 470         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 471                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 472 }
 473
 474 static void skl_init_clock_gating(struct drm_i915_private *i915)
 475 {
 476         gen9_init_clock_gating(i915);
 477
 478         /* WaDisableDopClockGating:skl */
 479         intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 480                          GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 481
 482         /* WAC6entrylatency:skl */
 483         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 484
 485         /*
 486          * WaFbcTurnOffFbcWatermark:skl
 487          * Display WA #0562: skl
 488          */
 489         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 490
 491         /*
 492          * WaFbcNukeOnHostModify:skl
 493          * Display WA #0873: skl
 494          */
 495         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 496                          0, DPFC_NUKE_ON_ANY_MODIFICATION);
 497
 498         /*
 499          * WaFbcHighMemBwCorruptionAvoidance:skl
 500          * Display WA #0883: skl
 501          */
 502         intel_uncore_rmw(&i915->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 503 }
 504
 505 static void bdw_init_clock_gating(struct drm_i915_private *i915)
 506 {
 507         enum pipe pipe;
 508
 509         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 510         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 511
 512         /* WaSwitchSolVfFArbitrationPriority:bdw */
 513         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 514
 515         /* WaPsrDPAMaskVBlankInSRD:bdw */
 516         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
 517
 518         for_each_pipe(i915, pipe) {
 519                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
 520                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 521                                  0, BDW_UNMASK_VBL_TO_REGS_IN_SRD);
 522         }
 523
 524         /* WaVSRefCountFullforceMissDisable:bdw */
 525         /* WaDSRefCountFullforceMissDisable:bdw */
 526         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 527                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 528
 529         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 530                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 531
 532         /* WaDisableSDEUnitClockGating:bdw */
 533         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 534
 535         /* WaProgramL3SqcReg1Default:bdw */
 536         gen8_set_l3sqc_credits(i915, 30, 2);
 537
 538         /* WaKVMNotificationOnConfigChange:bdw */
 539         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
 540                          0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 541
 542         lpt_init_clock_gating(i915);
 543
 544         /* WaDisableDopClockGating:bdw
 545          *
 546          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
 547          * clock gating.
 548          */
 549         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
 550 }
 551
 552 static void hsw_init_clock_gating(struct drm_i915_private *i915)
 553 {
 554         enum pipe pipe;
 555
 556         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 557         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 558
 559         /* WaPsrDPAMaskVBlankInSRD:hsw */
 560         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
 561
 562         for_each_pipe(i915, pipe) {
 563                 /* WaPsrDPRSUnmaskVBlankInSRD:hsw */
 564                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 565                                  0, HSW_UNMASK_VBL_TO_REGS_IN_SRD);
 566         }
 567
 568         /* This is required by WaCatErrorRejectionIssue:hsw */
 569         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 570                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 571
 572         /* WaSwitchSolVfFArbitrationPriority:hsw */
 573         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 574
 575         lpt_init_clock_gating(i915);
 576 }
 577
 578 static void ivb_init_clock_gating(struct drm_i915_private *i915)
 579 {
 580         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 581
 582         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
 583         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 584
 585         /* WaDisableBackToBackFlipFix:ivb */
 586         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 587                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 588                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 589
 590         if (IS_IVB_GT1(i915))
 591                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 592                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 593         else {
 594                 /* must write both registers */
 595                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 596                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 597                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
 598                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 599         }
 600
 601         /*
 602          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 603          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
 604          */
 605         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 606                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 607
 608         /* This is required by WaCatErrorRejectionIssue:ivb */
 609         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 610                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 611
 612         g4x_disable_trickle_feed(i915);
 613
 614         intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
 615                          GEN6_MBC_SNPCR_MED);
 616
 617         if (!HAS_PCH_NOP(i915))
 618                 cpt_init_clock_gating(i915);
 619
 620         gen6_check_mch_setup(i915);
 621 }
 622
 623 static void vlv_init_clock_gating(struct drm_i915_private *i915)
 624 {
 625         /* WaDisableBackToBackFlipFix:vlv */
 626         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 627                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 628                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 629
 630         /* WaDisableDopClockGating:vlv */
 631         intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 632                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 633
 634         /* This is required by WaCatErrorRejectionIssue:vlv */
 635         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 636                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 637
 638         /*
 639          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 640          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
 641          */
 642         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 643                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 644
 645         /* WaDisableL3Bank2xClockGate:vlv
 646          * Disabling L3 clock gating- MMIO 940c[25] = 1
 647          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
 648         intel_uncore_rmw(&i915->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 649
 650         /*
 651          * WaDisableVLVClockGating_VBIIssue:vlv
 652          * Disable clock gating on th GCFG unit to prevent a delay
 653          * in the reporting of vblank events.
 654          */
 655         intel_uncore_write(&i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 656 }
 657
 658 static void chv_init_clock_gating(struct drm_i915_private *i915)
 659 {
 660         /* WaVSRefCountFullforceMissDisable:chv */
 661         /* WaDSRefCountFullforceMissDisable:chv */
 662         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 663                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 664
 665         /* WaDisableSemaphoreAndSyncFlipWait:chv */
 666         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 667                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 668
 669         /* WaDisableCSUnitClockGating:chv */
 670         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 671
 672         /* WaDisableSDEUnitClockGating:chv */
 673         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 674
 675         /*
 676          * WaProgramL3SqcReg1Default:chv
 677          * See gfxspecs/Related Documents/Performance Guide/
 678          * LSQC Setting Recommendations.
 679          */
 680         gen8_set_l3sqc_credits(i915, 38, 2);
 681 }
 682
 683 static void g4x_init_clock_gating(struct drm_i915_private *i915)
 684 {
 685         u32 dspclk_gate;
 686
 687         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
 688         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
 689                            GS_UNIT_CLOCK_GATE_DISABLE |
 690                            CL_UNIT_CLOCK_GATE_DISABLE);
 691         intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
 692         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 693                 OVRUNIT_CLOCK_GATE_DISABLE |
 694                 OVCUNIT_CLOCK_GATE_DISABLE;
 695         if (IS_GM45(i915))
 696                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 697         intel_uncore_write(&i915->uncore, DSPCLK_GATE_D(i915), dspclk_gate);
 698
 699         g4x_disable_trickle_feed(i915);
 700 }
 701
 702 static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 703 {
 704         struct intel_uncore *uncore = &i915->uncore;
 705
 706         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 707         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
 708         intel_uncore_write(uncore, DSPCLK_GATE_D(i915), 0);
 709         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 710         intel_uncore_write16(uncore, DEUC, 0);
 711         intel_uncore_write(uncore,
 712                            MI_ARB_STATE,
 713                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 714 }
 715
 716 static void i965g_init_clock_gating(struct drm_i915_private *i915)
 717 {
 718         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
 719                            I965_RCC_CLOCK_GATE_DISABLE |
 720                            I965_RCPB_CLOCK_GATE_DISABLE |
 721                            I965_ISC_CLOCK_GATE_DISABLE |
 722                            I965_FBC_CLOCK_GATE_DISABLE);
 723         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
 724         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 725                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 726 }
 727
 728 static void gen3_init_clock_gating(struct drm_i915_private *i915)
 729 {
 730         u32 dstate = intel_uncore_read(&i915->uncore, D_STATE);
 731
 732         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
 733                 DSTATE_DOT_CLOCK_GATING;
 734         intel_uncore_write(&i915->uncore, D_STATE, dstate);
 735
 736         if (IS_PINEVIEW(i915))
 737                 intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 738                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
 739
 740         /* IIR "flip pending" means done if this bit is set */
 741         intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 742                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
 743
 744         /* interrupts should cause a wake up from C3 */
 745         intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
 746
 747         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 748         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 749                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
 750
 751         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 752                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 753 }
 754
 755 static void i85x_init_clock_gating(struct drm_i915_private *i915)
 756 {
 757         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 758
 759         /* interrupts should cause a wake up from C3 */
 760         intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
 761                            _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
 762
 763         intel_uncore_write(&i915->uncore, MEM_MODE,
 764                            _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 765
 766         /*
 767          * Have FBC ignore 3D activity since we use software
 768          * render tracking, and otherwise a pure 3D workload
 769          * (even if it just renders a single frame and then does
 770          * abosultely nothing) would not allow FBC to recompress
 771          * until a 2D blit occurs.
 772          */
 773         intel_uncore_write(&i915->uncore, SCPD0,
 774                            _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
 775 }
 776
 777 static void i830_init_clock_gating(struct drm_i915_private *i915)
 778 {
 779         intel_uncore_write(&i915->uncore, MEM_MODE,
 780                            _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
 781                            _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 782 }
 783
 784 void intel_clock_gating_init(struct drm_i915_private *i915)
 785 {
 786         i915->clock_gating_funcs->init_clock_gating(i915);
 787 }
 788
 789 static void nop_init_clock_gating(struct drm_i915_private *i915)
 790 {
 791         drm_dbg_kms(&i915->drm,
 792                     "No clock gating settings or workarounds applied.\n");
 793 }
 794
 795 #define CG_FUNCS(platform)                                              \
 796 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
 797         .init_clock_gating = platform##_init_clock_gating,              \
 798 }
 799
 800 CG_FUNCS(pvc);
 801 CG_FUNCS(dg2);
 802 CG_FUNCS(xehpsdv);
 803 CG_FUNCS(adlp);
 804 CG_FUNCS(gen12lp);
 805 CG_FUNCS(icl);
 806 CG_FUNCS(cfl);
 807 CG_FUNCS(skl);
 808 CG_FUNCS(kbl);
 809 CG_FUNCS(bxt);
 810 CG_FUNCS(glk);
 811 CG_FUNCS(bdw);
 812 CG_FUNCS(chv);
 813 CG_FUNCS(hsw);
 814 CG_FUNCS(ivb);
 815 CG_FUNCS(vlv);
 816 CG_FUNCS(gen6);
 817 CG_FUNCS(ilk);
 818 CG_FUNCS(g4x);
 819 CG_FUNCS(i965gm);
 820 CG_FUNCS(i965g);
 821 CG_FUNCS(gen3);
 822 CG_FUNCS(i85x);
 823 CG_FUNCS(i830);
 824 CG_FUNCS(nop);
 825 #undef CG_FUNCS
 826
 827 /**
 828  * intel_clock_gating_hooks_init - setup the clock gating hooks
 829  * @i915: device private
 830  *
 831  * Setup the hooks that configure which clocks of a given platform can be
 832  * gated and also apply various GT and display specific workarounds for these
 833  * platforms. Note that some GT specific workarounds are applied separately
 834  * when GPU contexts or batchbuffers start their execution.
 835  */
 836 void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
 837 {
 838         if (IS_METEORLAKE(i915))
 839                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 840         else if (IS_PONTEVECCHIO(i915))
 841                 i915->clock_gating_funcs = &pvc_clock_gating_funcs;
 842         else if (IS_DG2(i915))
 843                 i915->clock_gating_funcs = &dg2_clock_gating_funcs;
 844         else if (IS_XEHPSDV(i915))
 845                 i915->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
 846         else if (IS_ALDERLAKE_P(i915))
 847                 i915->clock_gating_funcs = &adlp_clock_gating_funcs;
 848         else if (GRAPHICS_VER(i915) == 12)
 849                 i915->clock_gating_funcs = &gen12lp_clock_gating_funcs;
 850         else if (GRAPHICS_VER(i915) == 11)
 851                 i915->clock_gating_funcs = &icl_clock_gating_funcs;
 852         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 853                 i915->clock_gating_funcs = &cfl_clock_gating_funcs;
 854         else if (IS_SKYLAKE(i915))
 855                 i915->clock_gating_funcs = &skl_clock_gating_funcs;
 856         else if (IS_KABYLAKE(i915))
 857                 i915->clock_gating_funcs = &kbl_clock_gating_funcs;
 858         else if (IS_BROXTON(i915))
 859                 i915->clock_gating_funcs = &bxt_clock_gating_funcs;
 860         else if (IS_GEMINILAKE(i915))
 861                 i915->clock_gating_funcs = &glk_clock_gating_funcs;
 862         else if (IS_BROADWELL(i915))
 863                 i915->clock_gating_funcs = &bdw_clock_gating_funcs;
 864         else if (IS_CHERRYVIEW(i915))
 865                 i915->clock_gating_funcs = &chv_clock_gating_funcs;
 866         else if (IS_HASWELL(i915))
 867                 i915->clock_gating_funcs = &hsw_clock_gating_funcs;
 868         else if (IS_IVYBRIDGE(i915))
 869                 i915->clock_gating_funcs = &ivb_clock_gating_funcs;
 870         else if (IS_VALLEYVIEW(i915))
 871                 i915->clock_gating_funcs = &vlv_clock_gating_funcs;
 872         else if (GRAPHICS_VER(i915) == 6)
 873                 i915->clock_gating_funcs = &gen6_clock_gating_funcs;
 874         else if (GRAPHICS_VER(i915) == 5)
 875                 i915->clock_gating_funcs = &ilk_clock_gating_funcs;
 876         else if (IS_G4X(i915))
 877                 i915->clock_gating_funcs = &g4x_clock_gating_funcs;
 878         else if (IS_I965GM(i915))
 879                 i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
 880         else if (IS_I965G(i915))
 881                 i915->clock_gating_funcs = &i965g_clock_gating_funcs;
 882         else if (GRAPHICS_VER(i915) == 3)
 883                 i915->clock_gating_funcs = &gen3_clock_gating_funcs;
 884         else if (IS_I85X(i915) || IS_I865G(i915))
 885                 i915->clock_gating_funcs = &i85x_clock_gating_funcs;
 886         else if (GRAPHICS_VER(i915) == 2)
 887                 i915->clock_gating_funcs = &i830_clock_gating_funcs;
 888         else {
 889                 MISSING_CASE(INTEL_DEVID(i915));
 890                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 891         }
 892 }