drivers/gpu/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *    Daniel Vetter <daniel.vetter@ffwll.ch>
  26  *
  27  */
  28
  29 #include <linux/pm_runtime.h>
  30 #include <linux/vgaarb.h>
  31
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 /**
  36  * DOC: runtime pm
  37  *
  38  * The i915 driver supports dynamic enabling and disabling of entire hardware
  39  * blocks at runtime. This is especially important on the display side where
  40  * software is supposed to control many power gates manually on recent hardware,
  41  * since on the GT side a lot of the power management is done by the hardware.
  42  * But even there some manual control at the device level is required.
  43  *
  44  * Since i915 supports a diverse set of platforms with a unified codebase and
  45  * hardware engineers just love to shuffle functionality around between power
  46  * domains there's a sizeable amount of indirection required. This file provides
  47  * generic functions to the driver for grabbing and releasing references for
  48  * abstract power domains. It then maps those to the actual power wells
  49  * present for a given platform.
  50  */
  51
  52 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  53                                          enum i915_power_well_id power_well_id);
  54
  55 static struct i915_power_well *
  56 lookup_power_well(struct drm_i915_private *dev_priv,
  57                   enum i915_power_well_id power_well_id);
  58
  59 const char *
  60 intel_display_power_domain_str(enum intel_display_power_domain domain)
  61 {
  62         switch (domain) {
  63         case POWER_DOMAIN_PIPE_A:
  64                 return "PIPE_A";
  65         case POWER_DOMAIN_PIPE_B:
  66                 return "PIPE_B";
  67         case POWER_DOMAIN_PIPE_C:
  68                 return "PIPE_C";
  69         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  70                 return "PIPE_A_PANEL_FITTER";
  71         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  72                 return "PIPE_B_PANEL_FITTER";
  73         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  74                 return "PIPE_C_PANEL_FITTER";
  75         case POWER_DOMAIN_TRANSCODER_A:
  76                 return "TRANSCODER_A";
  77         case POWER_DOMAIN_TRANSCODER_B:
  78                 return "TRANSCODER_B";
  79         case POWER_DOMAIN_TRANSCODER_C:
  80                 return "TRANSCODER_C";
  81         case POWER_DOMAIN_TRANSCODER_EDP:
  82                 return "TRANSCODER_EDP";
  83         case POWER_DOMAIN_TRANSCODER_DSI_A:
  84                 return "TRANSCODER_DSI_A";
  85         case POWER_DOMAIN_TRANSCODER_DSI_C:
  86                 return "TRANSCODER_DSI_C";
  87         case POWER_DOMAIN_PORT_DDI_A_LANES:
  88                 return "PORT_DDI_A_LANES";
  89         case POWER_DOMAIN_PORT_DDI_B_LANES:
  90                 return "PORT_DDI_B_LANES";
  91         case POWER_DOMAIN_PORT_DDI_C_LANES:
  92                 return "PORT_DDI_C_LANES";
  93         case POWER_DOMAIN_PORT_DDI_D_LANES:
  94                 return "PORT_DDI_D_LANES";
  95         case POWER_DOMAIN_PORT_DDI_E_LANES:
  96                 return "PORT_DDI_E_LANES";
  97         case POWER_DOMAIN_PORT_DDI_A_IO:
  98                 return "PORT_DDI_A_IO";
  99         case POWER_DOMAIN_PORT_DDI_B_IO:
 100                 return "PORT_DDI_B_IO";
 101         case POWER_DOMAIN_PORT_DDI_C_IO:
 102                 return "PORT_DDI_C_IO";
 103         case POWER_DOMAIN_PORT_DDI_D_IO:
 104                 return "PORT_DDI_D_IO";
 105         case POWER_DOMAIN_PORT_DDI_E_IO:
 106                 return "PORT_DDI_E_IO";
 107         case POWER_DOMAIN_PORT_DSI:
 108                 return "PORT_DSI";
 109         case POWER_DOMAIN_PORT_CRT:
 110                 return "PORT_CRT";
 111         case POWER_DOMAIN_PORT_OTHER:
 112                 return "PORT_OTHER";
 113         case POWER_DOMAIN_VGA:
 114                 return "VGA";
 115         case POWER_DOMAIN_AUDIO:
 116                 return "AUDIO";
 117         case POWER_DOMAIN_PLLS:
 118                 return "PLLS";
 119         case POWER_DOMAIN_AUX_A:
 120                 return "AUX_A";
 121         case POWER_DOMAIN_AUX_B:
 122                 return "AUX_B";
 123         case POWER_DOMAIN_AUX_C:
 124                 return "AUX_C";
 125         case POWER_DOMAIN_AUX_D:
 126                 return "AUX_D";
 127         case POWER_DOMAIN_GMBUS:
 128                 return "GMBUS";
 129         case POWER_DOMAIN_INIT:
 130                 return "INIT";
 131         case POWER_DOMAIN_MODESET:
 132                 return "MODESET";
 133         default:
 134                 MISSING_CASE(domain);
 135                 return "?";
 136         }
 137 }
 138
 139 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 140                                     struct i915_power_well *power_well)
 141 {
 142         DRM_DEBUG_KMS("enabling %s\n", power_well->name);
 143         power_well->ops->enable(dev_priv, power_well);
 144         power_well->hw_enabled = true;
 145 }
 146
 147 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 148                                      struct i915_power_well *power_well)
 149 {
 150         DRM_DEBUG_KMS("disabling %s\n", power_well->name);
 151         power_well->hw_enabled = false;
 152         power_well->ops->disable(dev_priv, power_well);
 153 }
 154
 155 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 156                                  struct i915_power_well *power_well)
 157 {
 158         if (!power_well->count++)
 159                 intel_power_well_enable(dev_priv, power_well);
 160 }
 161
 162 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 163                                  struct i915_power_well *power_well)
 164 {
 165         WARN(!power_well->count, "Use count on power well %s is already zero",
 166              power_well->name);
 167
 168         if (!--power_well->count)
 169                 intel_power_well_disable(dev_priv, power_well);
 170 }
 171
 172 /**
 173  * __intel_display_power_is_enabled - unlocked check for a power domain
 174  * @dev_priv: i915 device instance
 175  * @domain: power domain to check
 176  *
 177  * This is the unlocked version of intel_display_power_is_enabled() and should
 178  * only be used from error capture and recovery code where deadlocks are
 179  * possible.
 180  *
 181  * Returns:
 182  * True when the power domain is enabled, false otherwise.
 183  */
 184 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 185                                       enum intel_display_power_domain domain)
 186 {
 187         struct i915_power_well *power_well;
 188         bool is_enabled;
 189
 190         if (dev_priv->pm.suspended)
 191                 return false;
 192
 193         is_enabled = true;
 194
 195         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
 196                 if (power_well->always_on)
 197                         continue;
 198
 199                 if (!power_well->hw_enabled) {
 200                         is_enabled = false;
 201                         break;
 202                 }
 203         }
 204
 205         return is_enabled;
 206 }
 207
 208 /**
 209  * intel_display_power_is_enabled - check for a power domain
 210  * @dev_priv: i915 device instance
 211  * @domain: power domain to check
 212  *
 213  * This function can be used to check the hw power domain state. It is mostly
 214  * used in hardware state readout functions. Everywhere else code should rely
 215  * upon explicit power domain reference counting to ensure that the hardware
 216  * block is powered up before accessing it.
 217  *
 218  * Callers must hold the relevant modesetting locks to ensure that concurrent
 219  * threads can't disable the power well while the caller tries to read a few
 220  * registers.
 221  *
 222  * Returns:
 223  * True when the power domain is enabled, false otherwise.
 224  */
 225 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 226                                     enum intel_display_power_domain domain)
 227 {
 228         struct i915_power_domains *power_domains;
 229         bool ret;
 230
 231         power_domains = &dev_priv->power_domains;
 232
 233         mutex_lock(&power_domains->lock);
 234         ret = __intel_display_power_is_enabled(dev_priv, domain);
 235         mutex_unlock(&power_domains->lock);
 236
 237         return ret;
 238 }
 239
 240 /**
 241  * intel_display_set_init_power - set the initial power domain state
 242  * @dev_priv: i915 device instance
 243  * @enable: whether to enable or disable the initial power domain state
 244  *
 245  * For simplicity our driver load/unload and system suspend/resume code assumes
 246  * that all power domains are always enabled. This functions controls the state
 247  * of this little hack. While the initial power domain state is enabled runtime
 248  * pm is effectively disabled.
 249  */
 250 void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 251                                   bool enable)
 252 {
 253         if (dev_priv->power_domains.init_power_on == enable)
 254                 return;
 255
 256         if (enable)
 257                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 258         else
 259                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 260
 261         dev_priv->power_domains.init_power_on = enable;
 262 }
 263
 264 /*
 265  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 266  * when not needed anymore. We have 4 registers that can request the power well
 267  * to be enabled, and it will only be disabled if none of the registers is
 268  * requesting it to be enabled.
 269  */
 270 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 271                                        u8 irq_pipe_mask, bool has_vga)
 272 {
 273         struct pci_dev *pdev = dev_priv->drm.pdev;
 274
 275         /*
 276          * After we re-enable the power well, if we touch VGA register 0x3d5
 277          * we'll get unclaimed register interrupts. This stops after we write
 278          * anything to the VGA MSR register. The vgacon module uses this
 279          * register all the time, so if we unbind our driver and, as a
 280          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 281          * console_unlock(). So make here we touch the VGA MSR register, making
 282          * sure vgacon can keep working normally without triggering interrupts
 283          * and error messages.
 284          */
 285         if (has_vga) {
 286                 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 287                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 288                 vga_put(pdev, VGA_RSRC_LEGACY_IO);
 289         }
 290
 291         if (irq_pipe_mask)
 292                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 293 }
 294
 295 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 296                                        u8 irq_pipe_mask)
 297 {
 298         if (irq_pipe_mask)
 299                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 300 }
 301
 302
 303 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 304                                            struct i915_power_well *power_well)
 305 {
 306         enum i915_power_well_id id = power_well->id;
 307
 308         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 309         WARN_ON(intel_wait_for_register(dev_priv,
 310                                         HSW_PWR_WELL_CTL_DRIVER(id),
 311                                         HSW_PWR_WELL_CTL_STATE(id),
 312                                         HSW_PWR_WELL_CTL_STATE(id),
 313                                         1));
 314 }
 315
 316 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 317                                      enum i915_power_well_id id)
 318 {
 319         u32 req_mask = HSW_PWR_WELL_CTL_REQ(id);
 320         u32 ret;
 321
 322         ret = I915_READ(HSW_PWR_WELL_CTL_BIOS(id)) & req_mask ? 1 : 0;
 323         ret |= I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & req_mask ? 2 : 0;
 324         ret |= I915_READ(HSW_PWR_WELL_CTL_KVMR) & req_mask ? 4 : 0;
 325         ret |= I915_READ(HSW_PWR_WELL_CTL_DEBUG(id)) & req_mask ? 8 : 0;
 326
 327         return ret;
 328 }
 329
 330 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 331                                             struct i915_power_well *power_well)
 332 {
 333         enum i915_power_well_id id = power_well->id;
 334         bool disabled;
 335         u32 reqs;
 336
 337         /*
 338          * Bspec doesn't require waiting for PWs to get disabled, but still do
 339          * this for paranoia. The known cases where a PW will be forced on:
 340          * - a KVMR request on any power well via the KVMR request register
 341          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 342          *   DEBUG request registers
 343          * Skip the wait in case any of the request bits are set and print a
 344          * diagnostic message.
 345          */
 346         wait_for((disabled = !(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 347                                HSW_PWR_WELL_CTL_STATE(id))) ||
 348                  (reqs = hsw_power_well_requesters(dev_priv, id)), 1);
 349         if (disabled)
 350                 return;
 351
 352         DRM_DEBUG_KMS("%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 353                       power_well->name,
 354                       !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 355 }
 356
 357 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 358                                            enum skl_power_gate pg)
 359 {
 360         /* Timeout 5us for PG#0, for other PGs 1us */
 361         WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
 362                                         SKL_FUSE_PG_DIST_STATUS(pg),
 363                                         SKL_FUSE_PG_DIST_STATUS(pg), 1));
 364 }
 365
 366 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 367                                   struct i915_power_well *power_well)
 368 {
 369         enum i915_power_well_id id = power_well->id;
 370         bool wait_fuses = power_well->hsw.has_fuses;
 371         enum skl_power_gate uninitialized_var(pg);
 372         u32 val;
 373
 374         if (wait_fuses) {
 375                 pg = SKL_PW_TO_PG(id);
 376                 /*
 377                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 378                  * before enabling the power well and PW1/PG1's own fuse
 379                  * state after the enabling. For all other power wells with
 380                  * fuses we only have to wait for that PW/PG's fuse state
 381                  * after the enabling.
 382                  */
 383                 if (pg == SKL_PG1)
 384                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 385         }
 386
 387         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 388         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
 389         hsw_wait_for_power_well_enable(dev_priv, power_well);
 390
 391         if (wait_fuses)
 392                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 393
 394         hsw_power_well_post_enable(dev_priv, power_well->hsw.irq_pipe_mask,
 395                                    power_well->hsw.has_vga);
 396 }
 397
 398 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 399                                    struct i915_power_well *power_well)
 400 {
 401         enum i915_power_well_id id = power_well->id;
 402         u32 val;
 403
 404         hsw_power_well_pre_disable(dev_priv, power_well->hsw.irq_pipe_mask);
 405
 406         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 407         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
 408                    val & ~HSW_PWR_WELL_CTL_REQ(id));
 409         hsw_wait_for_power_well_disable(dev_priv, power_well);
 410 }
 411
 412 /*
 413  * We should only use the power well if we explicitly asked the hardware to
 414  * enable it, so check if it's enabled and also check if we've requested it to
 415  * be enabled.
 416  */
 417 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 418                                    struct i915_power_well *power_well)
 419 {
 420         enum i915_power_well_id id = power_well->id;
 421         u32 mask = HSW_PWR_WELL_CTL_REQ(id) | HSW_PWR_WELL_CTL_STATE(id);
 422
 423         return (I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & mask) == mask;
 424 }
 425
 426 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 427 {
 428         enum i915_power_well_id id = SKL_DISP_PW_2;
 429
 430         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 431                   "DC9 already programmed to be enabled.\n");
 432         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 433                   "DC5 still not disabled to enable DC9.\n");
 434         WARN_ONCE(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 435                   HSW_PWR_WELL_CTL_REQ(id),
 436                   "Power well 2 on.\n");
 437         WARN_ONCE(intel_irqs_enabled(dev_priv),
 438                   "Interrupts not disabled yet.\n");
 439
 440          /*
 441           * TODO: check for the following to verify the conditions to enter DC9
 442           * state are satisfied:
 443           * 1] Check relevant display engine registers to verify if mode set
 444           * disable sequence was followed.
 445           * 2] Check if display uninitialize sequence is initialized.
 446           */
 447 }
 448
 449 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 450 {
 451         WARN_ONCE(intel_irqs_enabled(dev_priv),
 452                   "Interrupts not disabled yet.\n");
 453         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 454                   "DC5 still not disabled.\n");
 455
 456          /*
 457           * TODO: check for the following to verify DC9 state was indeed
 458           * entered before programming to disable it:
 459           * 1] Check relevant display engine registers to verify if mode
 460           *  set disable sequence was followed.
 461           * 2] Check if display uninitialize sequence is initialized.
 462           */
 463 }
 464
 465 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 466                                 u32 state)
 467 {
 468         int rewrites = 0;
 469         int rereads = 0;
 470         u32 v;
 471
 472         I915_WRITE(DC_STATE_EN, state);
 473
 474         /* It has been observed that disabling the dc6 state sometimes
 475          * doesn't stick and dmc keeps returning old value. Make sure
 476          * the write really sticks enough times and also force rewrite until
 477          * we are confident that state is exactly what we want.
 478          */
 479         do  {
 480                 v = I915_READ(DC_STATE_EN);
 481
 482                 if (v != state) {
 483                         I915_WRITE(DC_STATE_EN, state);
 484                         rewrites++;
 485                         rereads = 0;
 486                 } else if (rereads++ > 5) {
 487                         break;
 488                 }
 489
 490         } while (rewrites < 100);
 491
 492         if (v != state)
 493                 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
 494                           state, v);
 495
 496         /* Most of the times we need one retry, avoid spam */
 497         if (rewrites > 1)
 498                 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
 499                               state, rewrites);
 500 }
 501
 502 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 503 {
 504         u32 mask;
 505
 506         mask = DC_STATE_EN_UPTO_DC5;
 507         if (IS_GEN9_LP(dev_priv))
 508                 mask |= DC_STATE_EN_DC9;
 509         else
 510                 mask |= DC_STATE_EN_UPTO_DC6;
 511
 512         return mask;
 513 }
 514
 515 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 516 {
 517         u32 val;
 518
 519         val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
 520
 521         DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
 522                       dev_priv->csr.dc_state, val);
 523         dev_priv->csr.dc_state = val;
 524 }
 525
 526 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 527 {
 528         uint32_t val;
 529         uint32_t mask;
 530
 531         if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
 532                 state &= dev_priv->csr.allowed_dc_mask;
 533
 534         val = I915_READ(DC_STATE_EN);
 535         mask = gen9_dc_mask(dev_priv);
 536         DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
 537                       val & mask, state);
 538
 539         /* Check if DMC is ignoring our DC state requests */
 540         if ((val & mask) != dev_priv->csr.dc_state)
 541                 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
 542                           dev_priv->csr.dc_state, val & mask);
 543
 544         val &= ~mask;
 545         val |= state;
 546
 547         gen9_write_dc_state(dev_priv, val);
 548
 549         dev_priv->csr.dc_state = val & mask;
 550 }
 551
 552 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 553 {
 554         assert_can_enable_dc9(dev_priv);
 555
 556         DRM_DEBUG_KMS("Enabling DC9\n");
 557
 558         intel_power_sequencer_reset(dev_priv);
 559         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 560 }
 561
 562 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 563 {
 564         assert_can_disable_dc9(dev_priv);
 565
 566         DRM_DEBUG_KMS("Disabling DC9\n");
 567
 568         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 569
 570         intel_pps_unlock_regs_wa(dev_priv);
 571 }
 572
 573 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 574 {
 575         WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
 576                   "CSR program storage start is NULL\n");
 577         WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
 578         WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 579 }
 580
 581 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 582 {
 583         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 584                                         SKL_DISP_PW_2);
 585
 586         WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 587
 588         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 589                   "DC5 already programmed to be enabled.\n");
 590         assert_rpm_wakelock_held(dev_priv);
 591
 592         assert_csr_loaded(dev_priv);
 593 }
 594
 595 void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 596 {
 597         assert_can_enable_dc5(dev_priv);
 598
 599         DRM_DEBUG_KMS("Enabling DC5\n");
 600
 601         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 602 }
 603
 604 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 605 {
 606         WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 607                   "Backlight is not disabled.\n");
 608         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 609                   "DC6 already programmed to be enabled.\n");
 610
 611         assert_csr_loaded(dev_priv);
 612 }
 613
 614 void skl_enable_dc6(struct drm_i915_private *dev_priv)
 615 {
 616         assert_can_enable_dc6(dev_priv);
 617
 618         DRM_DEBUG_KMS("Enabling DC6\n");
 619
 620         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 621
 622 }
 623
 624 void skl_disable_dc6(struct drm_i915_private *dev_priv)
 625 {
 626         DRM_DEBUG_KMS("Disabling DC6\n");
 627
 628         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 629 }
 630
 631 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 632                                    struct i915_power_well *power_well)
 633 {
 634         enum i915_power_well_id id = power_well->id;
 635         u32 mask = HSW_PWR_WELL_CTL_REQ(id);
 636         u32 bios_req = I915_READ(HSW_PWR_WELL_CTL_BIOS(id));
 637
 638         /* Take over the request bit if set by BIOS. */
 639         if (bios_req & mask) {
 640                 u32 drv_req = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 641
 642                 if (!(drv_req & mask))
 643                         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), drv_req | mask);
 644                 I915_WRITE(HSW_PWR_WELL_CTL_BIOS(id), bios_req & ~mask);
 645         }
 646 }
 647
 648 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 649                                            struct i915_power_well *power_well)
 650 {
 651         bxt_ddi_phy_init(dev_priv, power_well->bxt.phy);
 652 }
 653
 654 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 655                                             struct i915_power_well *power_well)
 656 {
 657         bxt_ddi_phy_uninit(dev_priv, power_well->bxt.phy);
 658 }
 659
 660 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 661                                             struct i915_power_well *power_well)
 662 {
 663         return bxt_ddi_phy_is_enabled(dev_priv, power_well->bxt.phy);
 664 }
 665
 666 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 667 {
 668         struct i915_power_well *power_well;
 669
 670         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 671         if (power_well->count > 0)
 672                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 673
 674         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 675         if (power_well->count > 0)
 676                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 677
 678         if (IS_GEMINILAKE(dev_priv)) {
 679                 power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
 680                 if (power_well->count > 0)
 681                         bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 682         }
 683 }
 684
 685 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 686                                            struct i915_power_well *power_well)
 687 {
 688         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 689 }
 690
 691 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 692 {
 693         u32 tmp = I915_READ(DBUF_CTL);
 694
 695         WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
 696              (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
 697              "Unexpected DBuf power power state (0x%08x)\n", tmp);
 698 }
 699
 700 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 701                                           struct i915_power_well *power_well)
 702 {
 703         struct intel_cdclk_state cdclk_state = {};
 704
 705         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 706
 707         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
 708         WARN_ON(!intel_cdclk_state_compare(&dev_priv->cdclk.hw, &cdclk_state));
 709
 710         gen9_assert_dbuf_enabled(dev_priv);
 711
 712         if (IS_GEN9_LP(dev_priv))
 713                 bxt_verify_ddi_phy_power_wells(dev_priv);
 714 }
 715
 716 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 717                                            struct i915_power_well *power_well)
 718 {
 719         if (!dev_priv->csr.dmc_payload)
 720                 return;
 721
 722         if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 723                 skl_enable_dc6(dev_priv);
 724         else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 725                 gen9_enable_dc5(dev_priv);
 726 }
 727
 728 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
 729                                          struct i915_power_well *power_well)
 730 {
 731 }
 732
 733 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 734                                            struct i915_power_well *power_well)
 735 {
 736 }
 737
 738 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 739                                              struct i915_power_well *power_well)
 740 {
 741         return true;
 742 }
 743
 744 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
 745                                          struct i915_power_well *power_well)
 746 {
 747         if ((I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
 748                 i830_enable_pipe(dev_priv, PIPE_A);
 749         if ((I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
 750                 i830_enable_pipe(dev_priv, PIPE_B);
 751 }
 752
 753 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
 754                                           struct i915_power_well *power_well)
 755 {
 756         i830_disable_pipe(dev_priv, PIPE_B);
 757         i830_disable_pipe(dev_priv, PIPE_A);
 758 }
 759
 760 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
 761                                           struct i915_power_well *power_well)
 762 {
 763         return I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
 764                 I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 765 }
 766
 767 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
 768                                           struct i915_power_well *power_well)
 769 {
 770         if (power_well->count > 0)
 771                 i830_pipes_power_well_enable(dev_priv, power_well);
 772         else
 773                 i830_pipes_power_well_disable(dev_priv, power_well);
 774 }
 775
 776 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 777                                struct i915_power_well *power_well, bool enable)
 778 {
 779         enum i915_power_well_id power_well_id = power_well->id;
 780         u32 mask;
 781         u32 state;
 782         u32 ctrl;
 783
 784         mask = PUNIT_PWRGT_MASK(power_well_id);
 785         state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 786                          PUNIT_PWRGT_PWR_GATE(power_well_id);
 787
 788         mutex_lock(&dev_priv->rps.hw_lock);
 789
 790 #define COND \
 791         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 792
 793         if (COND)
 794                 goto out;
 795
 796         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 797         ctrl &= ~mask;
 798         ctrl |= state;
 799         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 800
 801         if (wait_for(COND, 100))
 802                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 803                           state,
 804                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 805
 806 #undef COND
 807
 808 out:
 809         mutex_unlock(&dev_priv->rps.hw_lock);
 810 }
 811
 812 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 813                                   struct i915_power_well *power_well)
 814 {
 815         vlv_set_power_well(dev_priv, power_well, true);
 816 }
 817
 818 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 819                                    struct i915_power_well *power_well)
 820 {
 821         vlv_set_power_well(dev_priv, power_well, false);
 822 }
 823
 824 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 825                                    struct i915_power_well *power_well)
 826 {
 827         enum i915_power_well_id power_well_id = power_well->id;
 828         bool enabled = false;
 829         u32 mask;
 830         u32 state;
 831         u32 ctrl;
 832
 833         mask = PUNIT_PWRGT_MASK(power_well_id);
 834         ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
 835
 836         mutex_lock(&dev_priv->rps.hw_lock);
 837
 838         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
 839         /*
 840          * We only ever set the power-on and power-gate states, anything
 841          * else is unexpected.
 842          */
 843         WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
 844                 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
 845         if (state == ctrl)
 846                 enabled = true;
 847
 848         /*
 849          * A transient state at this point would mean some unexpected party
 850          * is poking at the power controls too.
 851          */
 852         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 853         WARN_ON(ctrl != state);
 854
 855         mutex_unlock(&dev_priv->rps.hw_lock);
 856
 857         return enabled;
 858 }
 859
 860 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 861 {
 862         u32 val;
 863
 864         /*
 865          * On driver load, a pipe may be active and driving a DSI display.
 866          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
 867          * (and never recovering) in this case. intel_dsi_post_disable() will
 868          * clear it when we turn off the display.
 869          */
 870         val = I915_READ(DSPCLK_GATE_D);
 871         val &= DPOUNIT_CLOCK_GATE_DISABLE;
 872         val |= VRHUNIT_CLOCK_GATE_DISABLE;
 873         I915_WRITE(DSPCLK_GATE_D, val);
 874
 875         /*
 876          * Disable trickle feed and enable pnd deadline calculation
 877          */
 878         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 879         I915_WRITE(CBR1_VLV, 0);
 880
 881         WARN_ON(dev_priv->rawclk_freq == 0);
 882
 883         I915_WRITE(RAWCLK_FREQ_VLV,
 884                    DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
 885 }
 886
 887 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 888 {
 889         struct intel_encoder *encoder;
 890         enum pipe pipe;
 891
 892         /*
 893          * Enable the CRI clock source so we can get at the
 894          * display and the reference clock for VGA
 895          * hotplug / manual detection. Supposedly DSI also
 896          * needs the ref clock up and running.
 897          *
 898          * CHV DPLL B/C have some issues if VGA mode is enabled.
 899          */
 900         for_each_pipe(dev_priv, pipe) {
 901                 u32 val = I915_READ(DPLL(pipe));
 902
 903                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
 904                 if (pipe != PIPE_A)
 905                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
 906
 907                 I915_WRITE(DPLL(pipe), val);
 908         }
 909
 910         vlv_init_display_clock_gating(dev_priv);
 911
 912         spin_lock_irq(&dev_priv->irq_lock);
 913         valleyview_enable_display_irqs(dev_priv);
 914         spin_unlock_irq(&dev_priv->irq_lock);
 915
 916         /*
 917          * During driver initialization/resume we can avoid restoring the
 918          * part of the HW/SW state that will be inited anyway explicitly.
 919          */
 920         if (dev_priv->power_domains.initializing)
 921                 return;
 922
 923         intel_hpd_init(dev_priv);
 924
 925         /* Re-enable the ADPA, if we have one */
 926         for_each_intel_encoder(&dev_priv->drm, encoder) {
 927                 if (encoder->type == INTEL_OUTPUT_ANALOG)
 928                         intel_crt_reset(&encoder->base);
 929         }
 930
 931         i915_redisable_vga_power_on(dev_priv);
 932
 933         intel_pps_unlock_regs_wa(dev_priv);
 934 }
 935
 936 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
 937 {
 938         spin_lock_irq(&dev_priv->irq_lock);
 939         valleyview_disable_display_irqs(dev_priv);
 940         spin_unlock_irq(&dev_priv->irq_lock);
 941
 942         /* make sure we're done processing display irqs */
 943         synchronize_irq(dev_priv->drm.irq);
 944
 945         intel_power_sequencer_reset(dev_priv);
 946
 947         /* Prevent us from re-enabling polling on accident in late suspend */
 948         if (!dev_priv->drm.dev->power.is_suspended)
 949                 intel_hpd_poll_init(dev_priv);
 950 }
 951
 952 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 953                                           struct i915_power_well *power_well)
 954 {
 955         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 956
 957         vlv_set_power_well(dev_priv, power_well, true);
 958
 959         vlv_display_power_well_init(dev_priv);
 960 }
 961
 962 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 963                                            struct i915_power_well *power_well)
 964 {
 965         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 966
 967         vlv_display_power_well_deinit(dev_priv);
 968
 969         vlv_set_power_well(dev_priv, power_well, false);
 970 }
 971
 972 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 973                                            struct i915_power_well *power_well)
 974 {
 975         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
 976
 977         /* since ref/cri clock was enabled */
 978         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 979
 980         vlv_set_power_well(dev_priv, power_well, true);
 981
 982         /*
 983          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
 984          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
 985          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
 986          *   b. The other bits such as sfr settings / modesel may all
 987          *      be set to 0.
 988          *
 989          * This should only be done on init and resume from S3 with
 990          * both PLLs disabled, or we risk losing DPIO and PLL
 991          * synchronization.
 992          */
 993         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
 994 }
 995
 996 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 997                                             struct i915_power_well *power_well)
 998 {
 999         enum pipe pipe;
1000
1001         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1002
1003         for_each_pipe(dev_priv, pipe)
1004                 assert_pll_disabled(dev_priv, pipe);
1005
1006         /* Assert common reset */
1007         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1008
1009         vlv_set_power_well(dev_priv, power_well, false);
1010 }
1011
1012 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1013
1014 static struct i915_power_well *
1015 lookup_power_well(struct drm_i915_private *dev_priv,
1016                   enum i915_power_well_id power_well_id)
1017 {
1018         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1019         int i;
1020
1021         for (i = 0; i < power_domains->power_well_count; i++) {
1022                 struct i915_power_well *power_well;
1023
1024                 power_well = &power_domains->power_wells[i];
1025                 if (power_well->id == power_well_id)
1026                         return power_well;
1027         }
1028
1029         return NULL;
1030 }
1031
1032 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1033
1034 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1035 {
1036         struct i915_power_well *cmn_bc =
1037                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1038         struct i915_power_well *cmn_d =
1039                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1040         u32 phy_control = dev_priv->chv_phy_control;
1041         u32 phy_status = 0;
1042         u32 phy_status_mask = 0xffffffff;
1043
1044         /*
1045          * The BIOS can leave the PHY is some weird state
1046          * where it doesn't fully power down some parts.
1047          * Disable the asserts until the PHY has been fully
1048          * reset (ie. the power well has been disabled at
1049          * least once).
1050          */
1051         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1052                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1053                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1054                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1055                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1056                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1057                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1058
1059         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1060                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1061                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1062                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1063
1064         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
1065                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1066
1067                 /* this assumes override is only used to enable lanes */
1068                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1069                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1070
1071                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1072                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1073
1074                 /* CL1 is on whenever anything is on in either channel */
1075                 if (BITS_SET(phy_control,
1076                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1077                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1078                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1079
1080                 /*
1081                  * The DPLLB check accounts for the pipe B + port A usage
1082                  * with CL2 powered up but all the lanes in the second channel
1083                  * powered down.
1084                  */
1085                 if (BITS_SET(phy_control,
1086                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1087                     (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1088                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1089
1090                 if (BITS_SET(phy_control,
1091                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1092                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1093                 if (BITS_SET(phy_control,
1094                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1095                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1096
1097                 if (BITS_SET(phy_control,
1098                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1099                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1100                 if (BITS_SET(phy_control,
1101                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1102                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1103         }
1104
1105         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
1106                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1107
1108                 /* this assumes override is only used to enable lanes */
1109                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1110                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1111
1112                 if (BITS_SET(phy_control,
1113                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1114                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1115
1116                 if (BITS_SET(phy_control,
1117                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1118                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1119                 if (BITS_SET(phy_control,
1120                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1121                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1122         }
1123
1124         phy_status &= phy_status_mask;
1125
1126         /*
1127          * The PHY may be busy with some initial calibration and whatnot,
1128          * so the power state can take a while to actually change.
1129          */
1130         if (intel_wait_for_register(dev_priv,
1131                                     DISPLAY_PHY_STATUS,
1132                                     phy_status_mask,
1133                                     phy_status,
1134                                     10))
1135                 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1136                           I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1137                            phy_status, dev_priv->chv_phy_control);
1138 }
1139
1140 #undef BITS_SET
1141
1142 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1143                                            struct i915_power_well *power_well)
1144 {
1145         enum dpio_phy phy;
1146         enum pipe pipe;
1147         uint32_t tmp;
1148
1149         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1150                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1151
1152         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1153                 pipe = PIPE_A;
1154                 phy = DPIO_PHY0;
1155         } else {
1156                 pipe = PIPE_C;
1157                 phy = DPIO_PHY1;
1158         }
1159
1160         /* since ref/cri clock was enabled */
1161         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1162         vlv_set_power_well(dev_priv, power_well, true);
1163
1164         /* Poll for phypwrgood signal */
1165         if (intel_wait_for_register(dev_priv,
1166                                     DISPLAY_PHY_STATUS,
1167                                     PHY_POWERGOOD(phy),
1168                                     PHY_POWERGOOD(phy),
1169                                     1))
1170                 DRM_ERROR("Display PHY %d is not power up\n", phy);
1171
1172         mutex_lock(&dev_priv->sb_lock);
1173
1174         /* Enable dynamic power down */
1175         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1176         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1177                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1178         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1179
1180         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1181                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1182                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1183                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1184         } else {
1185                 /*
1186                  * Force the non-existing CL2 off. BXT does this
1187                  * too, so maybe it saves some power even though
1188                  * CL2 doesn't exist?
1189                  */
1190                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1191                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1192                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1193         }
1194
1195         mutex_unlock(&dev_priv->sb_lock);
1196
1197         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1198         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1199
1200         DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1201                       phy, dev_priv->chv_phy_control);
1202
1203         assert_chv_phy_status(dev_priv);
1204 }
1205
1206 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1207                                             struct i915_power_well *power_well)
1208 {
1209         enum dpio_phy phy;
1210
1211         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1212                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1213
1214         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1215                 phy = DPIO_PHY0;
1216                 assert_pll_disabled(dev_priv, PIPE_A);
1217                 assert_pll_disabled(dev_priv, PIPE_B);
1218         } else {
1219                 phy = DPIO_PHY1;
1220                 assert_pll_disabled(dev_priv, PIPE_C);
1221         }
1222
1223         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1224         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1225
1226         vlv_set_power_well(dev_priv, power_well, false);
1227
1228         DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1229                       phy, dev_priv->chv_phy_control);
1230
1231         /* PHY is fully reset now, so we can enable the PHY state asserts */
1232         dev_priv->chv_phy_assert[phy] = true;
1233
1234         assert_chv_phy_status(dev_priv);
1235 }
1236
1237 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1238                                      enum dpio_channel ch, bool override, unsigned int mask)
1239 {
1240         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1241         u32 reg, val, expected, actual;
1242
1243         /*
1244          * The BIOS can leave the PHY is some weird state
1245          * where it doesn't fully power down some parts.
1246          * Disable the asserts until the PHY has been fully
1247          * reset (ie. the power well has been disabled at
1248          * least once).
1249          */
1250         if (!dev_priv->chv_phy_assert[phy])
1251                 return;
1252
1253         if (ch == DPIO_CH0)
1254                 reg = _CHV_CMN_DW0_CH0;
1255         else
1256                 reg = _CHV_CMN_DW6_CH1;
1257
1258         mutex_lock(&dev_priv->sb_lock);
1259         val = vlv_dpio_read(dev_priv, pipe, reg);
1260         mutex_unlock(&dev_priv->sb_lock);
1261
1262         /*
1263          * This assumes !override is only used when the port is disabled.
1264          * All lanes should power down even without the override when
1265          * the port is disabled.
1266          */
1267         if (!override || mask == 0xf) {
1268                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1269                 /*
1270                  * If CH1 common lane is not active anymore
1271                  * (eg. for pipe B DPLL) the entire channel will
1272                  * shut down, which causes the common lane registers
1273                  * to read as 0. That means we can't actually check
1274                  * the lane power down status bits, but as the entire
1275                  * register reads as 0 it's a good indication that the
1276                  * channel is indeed entirely powered down.
1277                  */
1278                 if (ch == DPIO_CH1 && val == 0)
1279                         expected = 0;
1280         } else if (mask != 0x0) {
1281                 expected = DPIO_ANYDL_POWERDOWN;
1282         } else {
1283                 expected = 0;
1284         }
1285
1286         if (ch == DPIO_CH0)
1287                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1288         else
1289                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1290         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1291
1292         WARN(actual != expected,
1293              "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1294              !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1295              !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1296              reg, val);
1297 }
1298
1299 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1300                           enum dpio_channel ch, bool override)
1301 {
1302         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1303         bool was_override;
1304
1305         mutex_lock(&power_domains->lock);
1306
1307         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1308
1309         if (override == was_override)
1310                 goto out;
1311
1312         if (override)
1313                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1314         else
1315                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1316
1317         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1318
1319         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1320                       phy, ch, dev_priv->chv_phy_control);
1321
1322         assert_chv_phy_status(dev_priv);
1323
1324 out:
1325         mutex_unlock(&power_domains->lock);
1326
1327         return was_override;
1328 }
1329
1330 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1331                              bool override, unsigned int mask)
1332 {
1333         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1334         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1335         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1336         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1337
1338         mutex_lock(&power_domains->lock);
1339
1340         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1341         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1342
1343         if (override)
1344                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1345         else
1346                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1347
1348         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1349
1350         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1351                       phy, ch, mask, dev_priv->chv_phy_control);
1352
1353         assert_chv_phy_status(dev_priv);
1354
1355         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1356
1357         mutex_unlock(&power_domains->lock);
1358 }
1359
1360 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1361                                         struct i915_power_well *power_well)
1362 {
1363         enum pipe pipe = PIPE_A;
1364         bool enabled;
1365         u32 state, ctrl;
1366
1367         mutex_lock(&dev_priv->rps.hw_lock);
1368
1369         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1370         /*
1371          * We only ever set the power-on and power-gate states, anything
1372          * else is unexpected.
1373          */
1374         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1375         enabled = state == DP_SSS_PWR_ON(pipe);
1376
1377         /*
1378          * A transient state at this point would mean some unexpected party
1379          * is poking at the power controls too.
1380          */
1381         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1382         WARN_ON(ctrl << 16 != state);
1383
1384         mutex_unlock(&dev_priv->rps.hw_lock);
1385
1386         return enabled;
1387 }
1388
1389 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1390                                     struct i915_power_well *power_well,
1391                                     bool enable)
1392 {
1393         enum pipe pipe = PIPE_A;
1394         u32 state;
1395         u32 ctrl;
1396
1397         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1398
1399         mutex_lock(&dev_priv->rps.hw_lock);
1400
1401 #define COND \
1402         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1403
1404         if (COND)
1405                 goto out;
1406
1407         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1408         ctrl &= ~DP_SSC_MASK(pipe);
1409         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1410         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1411
1412         if (wait_for(COND, 100))
1413                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1414                           state,
1415                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1416
1417 #undef COND
1418
1419 out:
1420         mutex_unlock(&dev_priv->rps.hw_lock);
1421 }
1422
1423 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1424                                        struct i915_power_well *power_well)
1425 {
1426         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1427
1428         chv_set_pipe_power_well(dev_priv, power_well, true);
1429
1430         vlv_display_power_well_init(dev_priv);
1431 }
1432
1433 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1434                                         struct i915_power_well *power_well)
1435 {
1436         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1437
1438         vlv_display_power_well_deinit(dev_priv);
1439
1440         chv_set_pipe_power_well(dev_priv, power_well, false);
1441 }
1442
1443 static void
1444 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1445                                  enum intel_display_power_domain domain)
1446 {
1447         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1448         struct i915_power_well *power_well;
1449
1450         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
1451                 intel_power_well_get(dev_priv, power_well);
1452
1453         power_domains->domain_use_count[domain]++;
1454 }
1455
1456 /**
1457  * intel_display_power_get - grab a power domain reference
1458  * @dev_priv: i915 device instance
1459  * @domain: power domain to reference
1460  *
1461  * This function grabs a power domain reference for @domain and ensures that the
1462  * power domain and all its parents are powered up. Therefore users should only
1463  * grab a reference to the innermost power domain they need.
1464  *
1465  * Any power domain reference obtained by this function must have a symmetric
1466  * call to intel_display_power_put() to release the reference again.
1467  */
1468 void intel_display_power_get(struct drm_i915_private *dev_priv,
1469                              enum intel_display_power_domain domain)
1470 {
1471         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1472
1473         intel_runtime_pm_get(dev_priv);
1474
1475         mutex_lock(&power_domains->lock);
1476
1477         __intel_display_power_get_domain(dev_priv, domain);
1478
1479         mutex_unlock(&power_domains->lock);
1480 }
1481
1482 /**
1483  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1484  * @dev_priv: i915 device instance
1485  * @domain: power domain to reference
1486  *
1487  * This function grabs a power domain reference for @domain and ensures that the
1488  * power domain and all its parents are powered up. Therefore users should only
1489  * grab a reference to the innermost power domain they need.
1490  *
1491  * Any power domain reference obtained by this function must have a symmetric
1492  * call to intel_display_power_put() to release the reference again.
1493  */
1494 bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1495                                         enum intel_display_power_domain domain)
1496 {
1497         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1498         bool is_enabled;
1499
1500         if (!intel_runtime_pm_get_if_in_use(dev_priv))
1501                 return false;
1502
1503         mutex_lock(&power_domains->lock);
1504
1505         if (__intel_display_power_is_enabled(dev_priv, domain)) {
1506                 __intel_display_power_get_domain(dev_priv, domain);
1507                 is_enabled = true;
1508         } else {
1509                 is_enabled = false;
1510         }
1511
1512         mutex_unlock(&power_domains->lock);
1513
1514         if (!is_enabled)
1515                 intel_runtime_pm_put(dev_priv);
1516
1517         return is_enabled;
1518 }
1519
1520 /**
1521  * intel_display_power_put - release a power domain reference
1522  * @dev_priv: i915 device instance
1523  * @domain: power domain to reference
1524  *
1525  * This function drops the power domain reference obtained by
1526  * intel_display_power_get() and might power down the corresponding hardware
1527  * block right away if this is the last reference.
1528  */
1529 void intel_display_power_put(struct drm_i915_private *dev_priv,
1530                              enum intel_display_power_domain domain)
1531 {
1532         struct i915_power_domains *power_domains;
1533         struct i915_power_well *power_well;
1534
1535         power_domains = &dev_priv->power_domains;
1536
1537         mutex_lock(&power_domains->lock);
1538
1539         WARN(!power_domains->domain_use_count[domain],
1540              "Use count on domain %s is already zero\n",
1541              intel_display_power_domain_str(domain));
1542         power_domains->domain_use_count[domain]--;
1543
1544         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
1545                 intel_power_well_put(dev_priv, power_well);
1546
1547         mutex_unlock(&power_domains->lock);
1548
1549         intel_runtime_pm_put(dev_priv);
1550 }
1551
1552 #define I830_PIPES_POWER_DOMAINS (              \
1553         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1554         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1555         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1556         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1557         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1558         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1559         BIT_ULL(POWER_DOMAIN_INIT))
1560
1561 #define VLV_DISPLAY_POWER_DOMAINS (             \
1562         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1563         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1564         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1565         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1566         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1567         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1568         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1569         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1570         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1571         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1572         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1573         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1574         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1575         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1576         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1577         BIT_ULL(POWER_DOMAIN_INIT))
1578
1579 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1580         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1581         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1582         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1583         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1584         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1585         BIT_ULL(POWER_DOMAIN_INIT))
1586
1587 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1588         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1589         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1590         BIT_ULL(POWER_DOMAIN_INIT))
1591
1592 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1593         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1594         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1595         BIT_ULL(POWER_DOMAIN_INIT))
1596
1597 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1598         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1599         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1600         BIT_ULL(POWER_DOMAIN_INIT))
1601
1602 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1603         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1604         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1605         BIT_ULL(POWER_DOMAIN_INIT))
1606
1607 #define CHV_DISPLAY_POWER_DOMAINS (             \
1608         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1609         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1610         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
1611         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1612         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1613         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1614         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1615         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1616         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
1617         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1618         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1619         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1620         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1621         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1622         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1623         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1624         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1625         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1626         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1627         BIT_ULL(POWER_DOMAIN_INIT))
1628
1629 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1630         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1631         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1632         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1633         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1634         BIT_ULL(POWER_DOMAIN_INIT))
1635
1636 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1637         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1638         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1639         BIT_ULL(POWER_DOMAIN_INIT))
1640
1641 #define HSW_DISPLAY_POWER_DOMAINS (                     \
1642         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1643         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1644         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
1645         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1646         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1647         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1648         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1649         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1650         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1651         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1652         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1653         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1654         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1655         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1656         BIT_ULL(POWER_DOMAIN_INIT))
1657
1658 #define BDW_DISPLAY_POWER_DOMAINS (                     \
1659         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1660         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1661         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1662         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1663         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1664         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1665         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1666         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1667         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1668         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1669         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1670         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1671         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1672         BIT_ULL(POWER_DOMAIN_INIT))
1673
1674 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1675         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1676         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1677         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1678         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1679         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1680         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1681         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1682         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1683         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1684         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1685         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
1686         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1687         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1688         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1689         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1690         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1691         BIT_ULL(POWER_DOMAIN_INIT))
1692 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
1693         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1694         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
1695         BIT_ULL(POWER_DOMAIN_INIT))
1696 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1697         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1698         BIT_ULL(POWER_DOMAIN_INIT))
1699 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1700         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1701         BIT_ULL(POWER_DOMAIN_INIT))
1702 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
1703         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1704         BIT_ULL(POWER_DOMAIN_INIT))
1705 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1706         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1707         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1708         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1709         BIT_ULL(POWER_DOMAIN_INIT))
1710
1711 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1712         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1713         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1714         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1715         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1716         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1717         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1718         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1719         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1720         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1721         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1722         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1723         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1724         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1725         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1726         BIT_ULL(POWER_DOMAIN_INIT))
1727 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1728         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1729         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1730         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1731         BIT_ULL(POWER_DOMAIN_INIT))
1732 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
1733         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1734         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1735         BIT_ULL(POWER_DOMAIN_INIT))
1736 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
1737         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1738         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1739         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1740         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1741         BIT_ULL(POWER_DOMAIN_INIT))
1742
1743 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1744         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1745         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1746         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1747         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1748         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1749         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1750         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1751         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1752         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1753         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1754         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1755         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1756         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1757         BIT_ULL(POWER_DOMAIN_INIT))
1758 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
1759         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
1760 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1761         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
1762 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1763         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
1764 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
1765         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1766         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1767         BIT_ULL(POWER_DOMAIN_INIT))
1768 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
1769         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1770         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1771         BIT_ULL(POWER_DOMAIN_INIT))
1772 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
1773         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1774         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1775         BIT_ULL(POWER_DOMAIN_INIT))
1776 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
1777         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
1778         BIT_ULL(POWER_DOMAIN_INIT))
1779 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
1780         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1781         BIT_ULL(POWER_DOMAIN_INIT))
1782 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
1783         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1784         BIT_ULL(POWER_DOMAIN_INIT))
1785 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1786         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1787         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1788         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1789         BIT_ULL(POWER_DOMAIN_INIT))
1790
1791 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1792         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1793         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1794         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1795         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1796         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1797         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1798         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1799         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1800         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1801         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1802         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1803         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1804         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1805         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1806         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1807         BIT_ULL(POWER_DOMAIN_INIT))
1808 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
1809         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1810         BIT_ULL(POWER_DOMAIN_INIT))
1811 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
1812         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1813         BIT_ULL(POWER_DOMAIN_INIT))
1814 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
1815         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1816         BIT_ULL(POWER_DOMAIN_INIT))
1817 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
1818         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1819         BIT_ULL(POWER_DOMAIN_INIT))
1820 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
1821         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1822         BIT_ULL(POWER_DOMAIN_INIT))
1823 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
1824         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1825         BIT_ULL(POWER_DOMAIN_INIT))
1826 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
1827         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1828         BIT_ULL(POWER_DOMAIN_INIT))
1829 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
1830         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1831         BIT_ULL(POWER_DOMAIN_INIT))
1832 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1833         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1834         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1835         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1836         BIT_ULL(POWER_DOMAIN_INIT))
1837
1838 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1839         .sync_hw = i9xx_power_well_sync_hw_noop,
1840         .enable = i9xx_always_on_power_well_noop,
1841         .disable = i9xx_always_on_power_well_noop,
1842         .is_enabled = i9xx_always_on_power_well_enabled,
1843 };
1844
1845 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
1846         .sync_hw = i9xx_power_well_sync_hw_noop,
1847         .enable = chv_pipe_power_well_enable,
1848         .disable = chv_pipe_power_well_disable,
1849         .is_enabled = chv_pipe_power_well_enabled,
1850 };
1851
1852 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1853         .sync_hw = i9xx_power_well_sync_hw_noop,
1854         .enable = chv_dpio_cmn_power_well_enable,
1855         .disable = chv_dpio_cmn_power_well_disable,
1856         .is_enabled = vlv_power_well_enabled,
1857 };
1858
1859 static struct i915_power_well i9xx_always_on_power_well[] = {
1860         {
1861                 .name = "always-on",
1862                 .always_on = 1,
1863                 .domains = POWER_DOMAIN_MASK,
1864                 .ops = &i9xx_always_on_power_well_ops,
1865                 .id = I915_DISP_PW_ALWAYS_ON,
1866         },
1867 };
1868
1869 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
1870         .sync_hw = i830_pipes_power_well_sync_hw,
1871         .enable = i830_pipes_power_well_enable,
1872         .disable = i830_pipes_power_well_disable,
1873         .is_enabled = i830_pipes_power_well_enabled,
1874 };
1875
1876 static struct i915_power_well i830_power_wells[] = {
1877         {
1878                 .name = "always-on",
1879                 .always_on = 1,
1880                 .domains = POWER_DOMAIN_MASK,
1881                 .ops = &i9xx_always_on_power_well_ops,
1882                 .id = I915_DISP_PW_ALWAYS_ON,
1883         },
1884         {
1885                 .name = "pipes",
1886                 .domains = I830_PIPES_POWER_DOMAINS,
1887                 .ops = &i830_pipes_power_well_ops,
1888                 .id = I830_DISP_PW_PIPES,
1889         },
1890 };
1891
1892 static const struct i915_power_well_ops hsw_power_well_ops = {
1893         .sync_hw = hsw_power_well_sync_hw,
1894         .enable = hsw_power_well_enable,
1895         .disable = hsw_power_well_disable,
1896         .is_enabled = hsw_power_well_enabled,
1897 };
1898
1899 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1900         .sync_hw = i9xx_power_well_sync_hw_noop,
1901         .enable = gen9_dc_off_power_well_enable,
1902         .disable = gen9_dc_off_power_well_disable,
1903         .is_enabled = gen9_dc_off_power_well_enabled,
1904 };
1905
1906 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1907         .sync_hw = i9xx_power_well_sync_hw_noop,
1908         .enable = bxt_dpio_cmn_power_well_enable,
1909         .disable = bxt_dpio_cmn_power_well_disable,
1910         .is_enabled = bxt_dpio_cmn_power_well_enabled,
1911 };
1912
1913 static struct i915_power_well hsw_power_wells[] = {
1914         {
1915                 .name = "always-on",
1916                 .always_on = 1,
1917                 .domains = POWER_DOMAIN_MASK,
1918                 .ops = &i9xx_always_on_power_well_ops,
1919                 .id = I915_DISP_PW_ALWAYS_ON,
1920         },
1921         {
1922                 .name = "display",
1923                 .domains = HSW_DISPLAY_POWER_DOMAINS,
1924                 .ops = &hsw_power_well_ops,
1925                 .id = HSW_DISP_PW_GLOBAL,
1926                 {
1927                         .hsw.has_vga = true,
1928                 },
1929         },
1930 };
1931
1932 static struct i915_power_well bdw_power_wells[] = {
1933         {
1934                 .name = "always-on",
1935                 .always_on = 1,
1936                 .domains = POWER_DOMAIN_MASK,
1937                 .ops = &i9xx_always_on_power_well_ops,
1938                 .id = I915_DISP_PW_ALWAYS_ON,
1939         },
1940         {
1941                 .name = "display",
1942                 .domains = BDW_DISPLAY_POWER_DOMAINS,
1943                 .ops = &hsw_power_well_ops,
1944                 .id = HSW_DISP_PW_GLOBAL,
1945                 {
1946                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
1947                         .hsw.has_vga = true,
1948                 },
1949         },
1950 };
1951
1952 static const struct i915_power_well_ops vlv_display_power_well_ops = {
1953         .sync_hw = i9xx_power_well_sync_hw_noop,
1954         .enable = vlv_display_power_well_enable,
1955         .disable = vlv_display_power_well_disable,
1956         .is_enabled = vlv_power_well_enabled,
1957 };
1958
1959 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1960         .sync_hw = i9xx_power_well_sync_hw_noop,
1961         .enable = vlv_dpio_cmn_power_well_enable,
1962         .disable = vlv_dpio_cmn_power_well_disable,
1963         .is_enabled = vlv_power_well_enabled,
1964 };
1965
1966 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1967         .sync_hw = i9xx_power_well_sync_hw_noop,
1968         .enable = vlv_power_well_enable,
1969         .disable = vlv_power_well_disable,
1970         .is_enabled = vlv_power_well_enabled,
1971 };
1972
1973 static struct i915_power_well vlv_power_wells[] = {
1974         {
1975                 .name = "always-on",
1976                 .always_on = 1,
1977                 .domains = POWER_DOMAIN_MASK,
1978                 .ops = &i9xx_always_on_power_well_ops,
1979                 .id = I915_DISP_PW_ALWAYS_ON,
1980         },
1981         {
1982                 .name = "display",
1983                 .domains = VLV_DISPLAY_POWER_DOMAINS,
1984                 .id = PUNIT_POWER_WELL_DISP2D,
1985                 .ops = &vlv_display_power_well_ops,
1986         },
1987         {
1988                 .name = "dpio-tx-b-01",
1989                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1990                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
1991                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
1992                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
1993                 .ops = &vlv_dpio_power_well_ops,
1994                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
1995         },
1996         {
1997                 .name = "dpio-tx-b-23",
1998                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1999                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2000                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2001                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2002                 .ops = &vlv_dpio_power_well_ops,
2003                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
2004         },
2005         {
2006                 .name = "dpio-tx-c-01",
2007                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2008                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2009                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2010                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2011                 .ops = &vlv_dpio_power_well_ops,
2012                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
2013         },
2014         {
2015                 .name = "dpio-tx-c-23",
2016                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2017                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2018                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2019                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2020                 .ops = &vlv_dpio_power_well_ops,
2021                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
2022         },
2023         {
2024                 .name = "dpio-common",
2025                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2026                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2027                 .ops = &vlv_dpio_cmn_power_well_ops,
2028         },
2029 };
2030
2031 static struct i915_power_well chv_power_wells[] = {
2032         {
2033                 .name = "always-on",
2034                 .always_on = 1,
2035                 .domains = POWER_DOMAIN_MASK,
2036                 .ops = &i9xx_always_on_power_well_ops,
2037                 .id = I915_DISP_PW_ALWAYS_ON,
2038         },
2039         {
2040                 .name = "display",
2041                 /*
2042                  * Pipe A power well is the new disp2d well. Pipe B and C
2043                  * power wells don't actually exist. Pipe A power well is
2044                  * required for any pipe to work.
2045                  */
2046                 .domains = CHV_DISPLAY_POWER_DOMAINS,
2047                 .id = CHV_DISP_PW_PIPE_A,
2048                 .ops = &chv_pipe_power_well_ops,
2049         },
2050         {
2051                 .name = "dpio-common-bc",
2052                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2053                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2054                 .ops = &chv_dpio_cmn_power_well_ops,
2055         },
2056         {
2057                 .name = "dpio-common-d",
2058                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2059                 .id = PUNIT_POWER_WELL_DPIO_CMN_D,
2060                 .ops = &chv_dpio_cmn_power_well_ops,
2061         },
2062 };
2063
2064 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2065                                          enum i915_power_well_id power_well_id)
2066 {
2067         struct i915_power_well *power_well;
2068         bool ret;
2069
2070         power_well = lookup_power_well(dev_priv, power_well_id);
2071         ret = power_well->ops->is_enabled(dev_priv, power_well);
2072
2073         return ret;
2074 }
2075
2076 static struct i915_power_well skl_power_wells[] = {
2077         {
2078                 .name = "always-on",
2079                 .always_on = 1,
2080                 .domains = POWER_DOMAIN_MASK,
2081                 .ops = &i9xx_always_on_power_well_ops,
2082                 .id = I915_DISP_PW_ALWAYS_ON,
2083         },
2084         {
2085                 .name = "power well 1",
2086                 /* Handled by the DMC firmware */
2087                 .domains = 0,
2088                 .ops = &hsw_power_well_ops,
2089                 .id = SKL_DISP_PW_1,
2090                 {
2091                         .hsw.has_fuses = true,
2092                 },
2093         },
2094         {
2095                 .name = "MISC IO power well",
2096                 /* Handled by the DMC firmware */
2097                 .domains = 0,
2098                 .ops = &hsw_power_well_ops,
2099                 .id = SKL_DISP_PW_MISC_IO,
2100         },
2101         {
2102                 .name = "DC off",
2103                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2104                 .ops = &gen9_dc_off_power_well_ops,
2105                 .id = SKL_DISP_PW_DC_OFF,
2106         },
2107         {
2108                 .name = "power well 2",
2109                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2110                 .ops = &hsw_power_well_ops,
2111                 .id = SKL_DISP_PW_2,
2112                 {
2113                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2114                         .hsw.has_vga = true,
2115                         .hsw.has_fuses = true,
2116                 },
2117         },
2118         {
2119                 .name = "DDI A/E IO power well",
2120                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
2121                 .ops = &hsw_power_well_ops,
2122                 .id = SKL_DISP_PW_DDI_A_E,
2123         },
2124         {
2125                 .name = "DDI B IO power well",
2126                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2127                 .ops = &hsw_power_well_ops,
2128                 .id = SKL_DISP_PW_DDI_B,
2129         },
2130         {
2131                 .name = "DDI C IO power well",
2132                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2133                 .ops = &hsw_power_well_ops,
2134                 .id = SKL_DISP_PW_DDI_C,
2135         },
2136         {
2137                 .name = "DDI D IO power well",
2138                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
2139                 .ops = &hsw_power_well_ops,
2140                 .id = SKL_DISP_PW_DDI_D,
2141         },
2142 };
2143
2144 static struct i915_power_well bxt_power_wells[] = {
2145         {
2146                 .name = "always-on",
2147                 .always_on = 1,
2148                 .domains = POWER_DOMAIN_MASK,
2149                 .ops = &i9xx_always_on_power_well_ops,
2150                 .id = I915_DISP_PW_ALWAYS_ON,
2151         },
2152         {
2153                 .name = "power well 1",
2154                 .domains = 0,
2155                 .ops = &hsw_power_well_ops,
2156                 .id = SKL_DISP_PW_1,
2157                 {
2158                         .hsw.has_fuses = true,
2159                 },
2160         },
2161         {
2162                 .name = "DC off",
2163                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2164                 .ops = &gen9_dc_off_power_well_ops,
2165                 .id = SKL_DISP_PW_DC_OFF,
2166         },
2167         {
2168                 .name = "power well 2",
2169                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2170                 .ops = &hsw_power_well_ops,
2171                 .id = SKL_DISP_PW_2,
2172                 {
2173                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2174                         .hsw.has_vga = true,
2175                         .hsw.has_fuses = true,
2176                 },
2177         },
2178         {
2179                 .name = "dpio-common-a",
2180                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2181                 .ops = &bxt_dpio_cmn_power_well_ops,
2182                 .id = BXT_DPIO_CMN_A,
2183                 {
2184                         .bxt.phy = DPIO_PHY1,
2185                 },
2186         },
2187         {
2188                 .name = "dpio-common-bc",
2189                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2190                 .ops = &bxt_dpio_cmn_power_well_ops,
2191                 .id = BXT_DPIO_CMN_BC,
2192                 {
2193                         .bxt.phy = DPIO_PHY0,
2194                 },
2195         },
2196 };
2197
2198 static struct i915_power_well glk_power_wells[] = {
2199         {
2200                 .name = "always-on",
2201                 .always_on = 1,
2202                 .domains = POWER_DOMAIN_MASK,
2203                 .ops = &i9xx_always_on_power_well_ops,
2204                 .id = I915_DISP_PW_ALWAYS_ON,
2205         },
2206         {
2207                 .name = "power well 1",
2208                 /* Handled by the DMC firmware */
2209                 .domains = 0,
2210                 .ops = &hsw_power_well_ops,
2211                 .id = SKL_DISP_PW_1,
2212                 {
2213                         .hsw.has_fuses = true,
2214                 },
2215         },
2216         {
2217                 .name = "DC off",
2218                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2219                 .ops = &gen9_dc_off_power_well_ops,
2220                 .id = SKL_DISP_PW_DC_OFF,
2221         },
2222         {
2223                 .name = "power well 2",
2224                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2225                 .ops = &hsw_power_well_ops,
2226                 .id = SKL_DISP_PW_2,
2227                 {
2228                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2229                         .hsw.has_vga = true,
2230                         .hsw.has_fuses = true,
2231                 },
2232         },
2233         {
2234                 .name = "dpio-common-a",
2235                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2236                 .ops = &bxt_dpio_cmn_power_well_ops,
2237                 .id = BXT_DPIO_CMN_A,
2238                 {
2239                         .bxt.phy = DPIO_PHY1,
2240                 },
2241         },
2242         {
2243                 .name = "dpio-common-b",
2244                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2245                 .ops = &bxt_dpio_cmn_power_well_ops,
2246                 .id = BXT_DPIO_CMN_BC,
2247                 {
2248                         .bxt.phy = DPIO_PHY0,
2249                 },
2250         },
2251         {
2252                 .name = "dpio-common-c",
2253                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2254                 .ops = &bxt_dpio_cmn_power_well_ops,
2255                 .id = GLK_DPIO_CMN_C,
2256                 {
2257                         .bxt.phy = DPIO_PHY2,
2258                 },
2259         },
2260         {
2261                 .name = "AUX A",
2262                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2263                 .ops = &hsw_power_well_ops,
2264                 .id = GLK_DISP_PW_AUX_A,
2265         },
2266         {
2267                 .name = "AUX B",
2268                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2269                 .ops = &hsw_power_well_ops,
2270                 .id = GLK_DISP_PW_AUX_B,
2271         },
2272         {
2273                 .name = "AUX C",
2274                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2275                 .ops = &hsw_power_well_ops,
2276                 .id = GLK_DISP_PW_AUX_C,
2277         },
2278         {
2279                 .name = "DDI A IO power well",
2280                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
2281                 .ops = &hsw_power_well_ops,
2282                 .id = GLK_DISP_PW_DDI_A,
2283         },
2284         {
2285                 .name = "DDI B IO power well",
2286                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2287                 .ops = &hsw_power_well_ops,
2288                 .id = SKL_DISP_PW_DDI_B,
2289         },
2290         {
2291                 .name = "DDI C IO power well",
2292                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2293                 .ops = &hsw_power_well_ops,
2294                 .id = SKL_DISP_PW_DDI_C,
2295         },
2296 };
2297
2298 static struct i915_power_well cnl_power_wells[] = {
2299         {
2300                 .name = "always-on",
2301                 .always_on = 1,
2302                 .domains = POWER_DOMAIN_MASK,
2303                 .ops = &i9xx_always_on_power_well_ops,
2304                 .id = I915_DISP_PW_ALWAYS_ON,
2305         },
2306         {
2307                 .name = "power well 1",
2308                 /* Handled by the DMC firmware */
2309                 .domains = 0,
2310                 .ops = &hsw_power_well_ops,
2311                 .id = SKL_DISP_PW_1,
2312                 {
2313                         .hsw.has_fuses = true,
2314                 },
2315         },
2316         {
2317                 .name = "AUX A",
2318                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
2319                 .ops = &hsw_power_well_ops,
2320                 .id = CNL_DISP_PW_AUX_A,
2321         },
2322         {
2323                 .name = "AUX B",
2324                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
2325                 .ops = &hsw_power_well_ops,
2326                 .id = CNL_DISP_PW_AUX_B,
2327         },
2328         {
2329                 .name = "AUX C",
2330                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
2331                 .ops = &hsw_power_well_ops,
2332                 .id = CNL_DISP_PW_AUX_C,
2333         },
2334         {
2335                 .name = "AUX D",
2336                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
2337                 .ops = &hsw_power_well_ops,
2338                 .id = CNL_DISP_PW_AUX_D,
2339         },
2340         {
2341                 .name = "DC off",
2342                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
2343                 .ops = &gen9_dc_off_power_well_ops,
2344                 .id = SKL_DISP_PW_DC_OFF,
2345         },
2346         {
2347                 .name = "power well 2",
2348                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2349                 .ops = &hsw_power_well_ops,
2350                 .id = SKL_DISP_PW_2,
2351                 {
2352                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2353                         .hsw.has_vga = true,
2354                         .hsw.has_fuses = true,
2355                 },
2356         },
2357         {
2358                 .name = "DDI A IO power well",
2359                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
2360                 .ops = &hsw_power_well_ops,
2361                 .id = CNL_DISP_PW_DDI_A,
2362         },
2363         {
2364                 .name = "DDI B IO power well",
2365                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
2366                 .ops = &hsw_power_well_ops,
2367                 .id = SKL_DISP_PW_DDI_B,
2368         },
2369         {
2370                 .name = "DDI C IO power well",
2371                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
2372                 .ops = &hsw_power_well_ops,
2373                 .id = SKL_DISP_PW_DDI_C,
2374         },
2375         {
2376                 .name = "DDI D IO power well",
2377                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
2378                 .ops = &hsw_power_well_ops,
2379                 .id = SKL_DISP_PW_DDI_D,
2380         },
2381 };
2382
2383 static int
2384 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2385                                    int disable_power_well)
2386 {
2387         if (disable_power_well >= 0)
2388                 return !!disable_power_well;
2389
2390         return 1;
2391 }
2392
2393 static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2394                                     int enable_dc)
2395 {
2396         uint32_t mask;
2397         int requested_dc;
2398         int max_dc;
2399
2400         if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
2401                 max_dc = 2;
2402                 mask = 0;
2403         } else if (IS_GEN9_LP(dev_priv)) {
2404                 max_dc = 1;
2405                 /*
2406                  * DC9 has a separate HW flow from the rest of the DC states,
2407                  * not depending on the DMC firmware. It's needed by system
2408                  * suspend/resume, so allow it unconditionally.
2409                  */
2410                 mask = DC_STATE_EN_DC9;
2411         } else {
2412                 max_dc = 0;
2413                 mask = 0;
2414         }
2415
2416         if (!i915.disable_power_well)
2417                 max_dc = 0;
2418
2419         if (enable_dc >= 0 && enable_dc <= max_dc) {
2420                 requested_dc = enable_dc;
2421         } else if (enable_dc == -1) {
2422                 requested_dc = max_dc;
2423         } else if (enable_dc > max_dc && enable_dc <= 2) {
2424                 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
2425                               enable_dc, max_dc);
2426                 requested_dc = max_dc;
2427         } else {
2428                 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
2429                 requested_dc = max_dc;
2430         }
2431
2432         if (requested_dc > 1)
2433                 mask |= DC_STATE_EN_UPTO_DC6;
2434         if (requested_dc > 0)
2435                 mask |= DC_STATE_EN_UPTO_DC5;
2436
2437         DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
2438
2439         return mask;
2440 }
2441
2442 static void assert_power_well_ids_unique(struct drm_i915_private *dev_priv)
2443 {
2444         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2445         u64 power_well_ids;
2446         int i;
2447
2448         power_well_ids = 0;
2449         for (i = 0; i < power_domains->power_well_count; i++) {
2450                 enum i915_power_well_id id = power_domains->power_wells[i].id;
2451
2452                 WARN_ON(id >= sizeof(power_well_ids) * 8);
2453                 WARN_ON(power_well_ids & BIT_ULL(id));
2454                 power_well_ids |= BIT_ULL(id);
2455         }
2456 }
2457
2458 #define set_power_wells(power_domains, __power_wells) ({                \
2459         (power_domains)->power_wells = (__power_wells);                 \
2460         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
2461 })
2462
2463 /**
2464  * intel_power_domains_init - initializes the power domain structures
2465  * @dev_priv: i915 device instance
2466  *
2467  * Initializes the power domain structures for @dev_priv depending upon the
2468  * supported platform.
2469  */
2470 int intel_power_domains_init(struct drm_i915_private *dev_priv)
2471 {
2472         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2473
2474         i915.disable_power_well = sanitize_disable_power_well_option(dev_priv,
2475                                                      i915.disable_power_well);
2476         dev_priv->csr.allowed_dc_mask = get_allowed_dc_mask(dev_priv,
2477                                                             i915.enable_dc);
2478
2479         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
2480
2481         mutex_init(&power_domains->lock);
2482
2483         /*
2484          * The enabling order will be from lower to higher indexed wells,
2485          * the disabling order is reversed.
2486          */
2487         if (IS_HASWELL(dev_priv)) {
2488                 set_power_wells(power_domains, hsw_power_wells);
2489         } else if (IS_BROADWELL(dev_priv)) {
2490                 set_power_wells(power_domains, bdw_power_wells);
2491         } else if (IS_GEN9_BC(dev_priv)) {
2492                 set_power_wells(power_domains, skl_power_wells);
2493         } else if (IS_CANNONLAKE(dev_priv)) {
2494                 set_power_wells(power_domains, cnl_power_wells);
2495         } else if (IS_BROXTON(dev_priv)) {
2496                 set_power_wells(power_domains, bxt_power_wells);
2497         } else if (IS_GEMINILAKE(dev_priv)) {
2498                 set_power_wells(power_domains, glk_power_wells);
2499         } else if (IS_CHERRYVIEW(dev_priv)) {
2500                 set_power_wells(power_domains, chv_power_wells);
2501         } else if (IS_VALLEYVIEW(dev_priv)) {
2502                 set_power_wells(power_domains, vlv_power_wells);
2503         } else if (IS_I830(dev_priv)) {
2504                 set_power_wells(power_domains, i830_power_wells);
2505         } else {
2506                 set_power_wells(power_domains, i9xx_always_on_power_well);
2507         }
2508
2509         assert_power_well_ids_unique(dev_priv);
2510
2511         return 0;
2512 }
2513
2514 /**
2515  * intel_power_domains_fini - finalizes the power domain structures
2516  * @dev_priv: i915 device instance
2517  *
2518  * Finalizes the power domain structures for @dev_priv depending upon the
2519  * supported platform. This function also disables runtime pm and ensures that
2520  * the device stays powered up so that the driver can be reloaded.
2521  */
2522 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
2523 {
2524         struct device *kdev = &dev_priv->drm.pdev->dev;
2525
2526         /*
2527          * The i915.ko module is still not prepared to be loaded when
2528          * the power well is not enabled, so just enable it in case
2529          * we're going to unload/reload.
2530          * The following also reacquires the RPM reference the core passed
2531          * to the driver during loading, which is dropped in
2532          * intel_runtime_pm_enable(). We have to hand back the control of the
2533          * device to the core with this reference held.
2534          */
2535         intel_display_set_init_power(dev_priv, true);
2536
2537         /* Remove the refcount we took to keep power well support disabled. */
2538         if (!i915.disable_power_well)
2539                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2540
2541         /*
2542          * Remove the refcount we took in intel_runtime_pm_enable() in case
2543          * the platform doesn't support runtime PM.
2544          */
2545         if (!HAS_RUNTIME_PM(dev_priv))
2546                 pm_runtime_put(kdev);
2547 }
2548
2549 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
2550 {
2551         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2552         struct i915_power_well *power_well;
2553
2554         mutex_lock(&power_domains->lock);
2555         for_each_power_well(dev_priv, power_well) {
2556                 power_well->ops->sync_hw(dev_priv, power_well);
2557                 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
2558                                                                      power_well);
2559         }
2560         mutex_unlock(&power_domains->lock);
2561 }
2562
2563 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
2564 {
2565         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
2566         POSTING_READ(DBUF_CTL);
2567
2568         udelay(10);
2569
2570         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
2571                 DRM_ERROR("DBuf power enable timeout\n");
2572 }
2573
2574 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
2575 {
2576         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
2577         POSTING_READ(DBUF_CTL);
2578
2579         udelay(10);
2580
2581         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
2582                 DRM_ERROR("DBuf power disable timeout!\n");
2583 }
2584
2585 static void skl_display_core_init(struct drm_i915_private *dev_priv,
2586                                    bool resume)
2587 {
2588         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2589         struct i915_power_well *well;
2590         uint32_t val;
2591
2592         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2593
2594         /* enable PCH reset handshake */
2595         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2596         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
2597
2598         /* enable PG1 and Misc I/O */
2599         mutex_lock(&power_domains->lock);
2600
2601         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2602         intel_power_well_enable(dev_priv, well);
2603
2604         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2605         intel_power_well_enable(dev_priv, well);
2606
2607         mutex_unlock(&power_domains->lock);
2608
2609         skl_init_cdclk(dev_priv);
2610
2611         gen9_dbuf_enable(dev_priv);
2612
2613         if (resume && dev_priv->csr.dmc_payload)
2614                 intel_csr_load_program(dev_priv);
2615 }
2616
2617 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
2618 {
2619         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2620         struct i915_power_well *well;
2621
2622         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2623
2624         gen9_dbuf_disable(dev_priv);
2625
2626         skl_uninit_cdclk(dev_priv);
2627
2628         /* The spec doesn't call for removing the reset handshake flag */
2629         /* disable PG1 and Misc I/O */
2630
2631         mutex_lock(&power_domains->lock);
2632
2633         /*
2634          * BSpec says to keep the MISC IO power well enabled here, only
2635          * remove our request for power well 1.
2636          * Note that even though the driver's request is removed power well 1
2637          * may stay enabled after this due to DMC's own request on it.
2638          */
2639         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2640         intel_power_well_disable(dev_priv, well);
2641
2642         mutex_unlock(&power_domains->lock);
2643
2644         usleep_range(10, 30);           /* 10 us delay per Bspec */
2645 }
2646
2647 void bxt_display_core_init(struct drm_i915_private *dev_priv,
2648                            bool resume)
2649 {
2650         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2651         struct i915_power_well *well;
2652         uint32_t val;
2653
2654         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2655
2656         /*
2657          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
2658          * or else the reset will hang because there is no PCH to respond.
2659          * Move the handshake programming to initialization sequence.
2660          * Previously was left up to BIOS.
2661          */
2662         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2663         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
2664         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2665
2666         /* Enable PG1 */
2667         mutex_lock(&power_domains->lock);
2668
2669         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2670         intel_power_well_enable(dev_priv, well);
2671
2672         mutex_unlock(&power_domains->lock);
2673
2674         bxt_init_cdclk(dev_priv);
2675
2676         gen9_dbuf_enable(dev_priv);
2677
2678         if (resume && dev_priv->csr.dmc_payload)
2679                 intel_csr_load_program(dev_priv);
2680 }
2681
2682 void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
2683 {
2684         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2685         struct i915_power_well *well;
2686
2687         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2688
2689         gen9_dbuf_disable(dev_priv);
2690
2691         bxt_uninit_cdclk(dev_priv);
2692
2693         /* The spec doesn't call for removing the reset handshake flag */
2694
2695         /*
2696          * Disable PW1 (PG1).
2697          * Note that even though the driver's request is removed power well 1
2698          * may stay enabled after this due to DMC's own request on it.
2699          */
2700         mutex_lock(&power_domains->lock);
2701
2702         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2703         intel_power_well_disable(dev_priv, well);
2704
2705         mutex_unlock(&power_domains->lock);
2706
2707         usleep_range(10, 30);           /* 10 us delay per Bspec */
2708 }
2709
2710 #define CNL_PROCMON_IDX(val) \
2711         (((val) & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) >> VOLTAGE_INFO_SHIFT)
2712 #define NUM_CNL_PROCMON \
2713         (CNL_PROCMON_IDX(VOLTAGE_INFO_MASK | PROCESS_INFO_MASK) + 1)
2714
2715 static const struct cnl_procmon {
2716         u32 dw1, dw9, dw10;
2717 } cnl_procmon_values[NUM_CNL_PROCMON] = {
2718         [CNL_PROCMON_IDX(VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0)] =
2719                 { .dw1 = 0x00 << 16, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, },
2720         [CNL_PROCMON_IDX(VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0)] =
2721                 { .dw1 = 0x00 << 16, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, },
2722         [CNL_PROCMON_IDX(VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1)] =
2723                 { .dw1 = 0x00 << 16, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, },
2724         [CNL_PROCMON_IDX(VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0)] =
2725                 { .dw1 = 0x00 << 16, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, },
2726         [CNL_PROCMON_IDX(VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1)] =
2727                 { .dw1 = 0x44 << 16, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, },
2728 };
2729
2730 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
2731 {
2732         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2733         const struct cnl_procmon *procmon;
2734         struct i915_power_well *well;
2735         u32 val;
2736
2737         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2738
2739         /* 1. Enable PCH Reset Handshake */
2740         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2741         val |= RESET_PCH_HANDSHAKE_ENABLE;
2742         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2743
2744         /* 2. Enable Comp */
2745         val = I915_READ(CHICKEN_MISC_2);
2746         val &= ~CNL_COMP_PWR_DOWN;
2747         I915_WRITE(CHICKEN_MISC_2, val);
2748
2749         val = I915_READ(CNL_PORT_COMP_DW3);
2750         procmon = &cnl_procmon_values[CNL_PROCMON_IDX(val)];
2751
2752         WARN_ON(procmon->dw10 == 0);
2753
2754         val = I915_READ(CNL_PORT_COMP_DW1);
2755         val &= ~((0xff << 16) | 0xff);
2756         val |= procmon->dw1;
2757         I915_WRITE(CNL_PORT_COMP_DW1, val);
2758
2759         I915_WRITE(CNL_PORT_COMP_DW9, procmon->dw9);
2760         I915_WRITE(CNL_PORT_COMP_DW10, procmon->dw10);
2761
2762         val = I915_READ(CNL_PORT_COMP_DW0);
2763         val |= COMP_INIT;
2764         I915_WRITE(CNL_PORT_COMP_DW0, val);
2765
2766         /* 3. */
2767         val = I915_READ(CNL_PORT_CL1CM_DW5);
2768         val |= CL_POWER_DOWN_ENABLE;
2769         I915_WRITE(CNL_PORT_CL1CM_DW5, val);
2770
2771         /*
2772          * 4. Enable Power Well 1 (PG1).
2773          *    The AUX IO power wells will be enabled on demand.
2774          */
2775         mutex_lock(&power_domains->lock);
2776         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2777         intel_power_well_enable(dev_priv, well);
2778         mutex_unlock(&power_domains->lock);
2779
2780         /* 5. Enable CD clock */
2781         cnl_init_cdclk(dev_priv);
2782
2783         /* 6. Enable DBUF */
2784         gen9_dbuf_enable(dev_priv);
2785
2786         if (resume && dev_priv->csr.dmc_payload)
2787                 intel_csr_load_program(dev_priv);
2788 }
2789
2790 #undef CNL_PROCMON_IDX
2791 #undef NUM_CNL_PROCMON
2792
2793 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
2794 {
2795         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2796         struct i915_power_well *well;
2797         u32 val;
2798
2799         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2800
2801         /* 1. Disable all display engine functions -> aready done */
2802
2803         /* 2. Disable DBUF */
2804         gen9_dbuf_disable(dev_priv);
2805
2806         /* 3. Disable CD clock */
2807         cnl_uninit_cdclk(dev_priv);
2808
2809         /*
2810          * 4. Disable Power Well 1 (PG1).
2811          *    The AUX IO power wells are toggled on demand, so they are already
2812          *    disabled at this point.
2813          */
2814         mutex_lock(&power_domains->lock);
2815         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2816         intel_power_well_disable(dev_priv, well);
2817         mutex_unlock(&power_domains->lock);
2818
2819         usleep_range(10, 30);           /* 10 us delay per Bspec */
2820
2821         /* 5. Disable Comp */
2822         val = I915_READ(CHICKEN_MISC_2);
2823         val |= CNL_COMP_PWR_DOWN;
2824         I915_WRITE(CHICKEN_MISC_2, val);
2825 }
2826
2827 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
2828 {
2829         struct i915_power_well *cmn_bc =
2830                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2831         struct i915_power_well *cmn_d =
2832                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
2833
2834         /*
2835          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
2836          * workaround never ever read DISPLAY_PHY_CONTROL, and
2837          * instead maintain a shadow copy ourselves. Use the actual
2838          * power well state and lane status to reconstruct the
2839          * expected initial value.
2840          */
2841         dev_priv->chv_phy_control =
2842                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
2843                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
2844                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
2845                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
2846                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
2847
2848         /*
2849          * If all lanes are disabled we leave the override disabled
2850          * with all power down bits cleared to match the state we
2851          * would use after disabling the port. Otherwise enable the
2852          * override and set the lane powerdown bits accding to the
2853          * current lane status.
2854          */
2855         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
2856                 uint32_t status = I915_READ(DPLL(PIPE_A));
2857                 unsigned int mask;
2858
2859                 mask = status & DPLL_PORTB_READY_MASK;
2860                 if (mask == 0xf)
2861                         mask = 0x0;
2862                 else
2863                         dev_priv->chv_phy_control |=
2864                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
2865
2866                 dev_priv->chv_phy_control |=
2867                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
2868
2869                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
2870                 if (mask == 0xf)
2871                         mask = 0x0;
2872                 else
2873                         dev_priv->chv_phy_control |=
2874                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
2875
2876                 dev_priv->chv_phy_control |=
2877                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
2878
2879                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
2880
2881                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
2882         } else {
2883                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
2884         }
2885
2886         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
2887                 uint32_t status = I915_READ(DPIO_PHY_STATUS);
2888                 unsigned int mask;
2889
2890                 mask = status & DPLL_PORTD_READY_MASK;
2891
2892                 if (mask == 0xf)
2893                         mask = 0x0;
2894                 else
2895                         dev_priv->chv_phy_control |=
2896                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
2897
2898                 dev_priv->chv_phy_control |=
2899                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
2900
2901                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
2902
2903                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
2904         } else {
2905                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
2906         }
2907
2908         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
2909
2910         DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
2911                       dev_priv->chv_phy_control);
2912 }
2913
2914 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
2915 {
2916         struct i915_power_well *cmn =
2917                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2918         struct i915_power_well *disp2d =
2919                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
2920
2921         /* If the display might be already active skip this */
2922         if (cmn->ops->is_enabled(dev_priv, cmn) &&
2923             disp2d->ops->is_enabled(dev_priv, disp2d) &&
2924             I915_READ(DPIO_CTL) & DPIO_CMNRST)
2925                 return;
2926
2927         DRM_DEBUG_KMS("toggling display PHY side reset\n");
2928
2929         /* cmnlane needs DPLL registers */
2930         disp2d->ops->enable(dev_priv, disp2d);
2931
2932         /*
2933          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
2934          * Need to assert and de-assert PHY SB reset by gating the
2935          * common lane power, then un-gating it.
2936          * Simply ungating isn't enough to reset the PHY enough to get
2937          * ports and lanes running.
2938          */
2939         cmn->ops->disable(dev_priv, cmn);
2940 }
2941
2942 /**
2943  * intel_power_domains_init_hw - initialize hardware power domain state
2944  * @dev_priv: i915 device instance
2945  * @resume: Called from resume code paths or not
2946  *
2947  * This function initializes the hardware power domain state and enables all
2948  * power wells belonging to the INIT power domain. Power wells in other
2949  * domains (and not in the INIT domain) are referenced or disabled during the
2950  * modeset state HW readout. After that the reference count of each power well
2951  * must match its HW enabled state, see intel_power_domains_verify_state().
2952  */
2953 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
2954 {
2955         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2956
2957         power_domains->initializing = true;
2958
2959         if (IS_CANNONLAKE(dev_priv)) {
2960                 cnl_display_core_init(dev_priv, resume);
2961         } else if (IS_GEN9_BC(dev_priv)) {
2962                 skl_display_core_init(dev_priv, resume);
2963         } else if (IS_GEN9_LP(dev_priv)) {
2964                 bxt_display_core_init(dev_priv, resume);
2965         } else if (IS_CHERRYVIEW(dev_priv)) {
2966                 mutex_lock(&power_domains->lock);
2967                 chv_phy_control_init(dev_priv);
2968                 mutex_unlock(&power_domains->lock);
2969         } else if (IS_VALLEYVIEW(dev_priv)) {
2970                 mutex_lock(&power_domains->lock);
2971                 vlv_cmnlane_wa(dev_priv);
2972                 mutex_unlock(&power_domains->lock);
2973         }
2974
2975         /* For now, we need the power well to be always enabled. */
2976         intel_display_set_init_power(dev_priv, true);
2977         /* Disable power support if the user asked so. */
2978         if (!i915.disable_power_well)
2979                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
2980         intel_power_domains_sync_hw(dev_priv);
2981         power_domains->initializing = false;
2982 }
2983
2984 /**
2985  * intel_power_domains_suspend - suspend power domain state
2986  * @dev_priv: i915 device instance
2987  *
2988  * This function prepares the hardware power domain state before entering
2989  * system suspend. It must be paired with intel_power_domains_init_hw().
2990  */
2991 void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
2992 {
2993         /*
2994          * Even if power well support was disabled we still want to disable
2995          * power wells while we are system suspended.
2996          */
2997         if (!i915.disable_power_well)
2998                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2999
3000         if (IS_CANNONLAKE(dev_priv))
3001                 cnl_display_core_uninit(dev_priv);
3002         else if (IS_GEN9_BC(dev_priv))
3003                 skl_display_core_uninit(dev_priv);
3004         else if (IS_GEN9_LP(dev_priv))
3005                 bxt_display_core_uninit(dev_priv);
3006 }
3007
3008 static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv)
3009 {
3010         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3011         struct i915_power_well *power_well;
3012
3013         for_each_power_well(dev_priv, power_well) {
3014                 enum intel_display_power_domain domain;
3015
3016                 DRM_DEBUG_DRIVER("%-25s %d\n",
3017                                  power_well->name, power_well->count);
3018
3019                 for_each_power_domain(domain, power_well->domains)
3020                         DRM_DEBUG_DRIVER("  %-23s %d\n",
3021                                          intel_display_power_domain_str(domain),
3022                                          power_domains->domain_use_count[domain]);
3023         }
3024 }
3025
3026 /**
3027  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
3028  * @dev_priv: i915 device instance
3029  *
3030  * Verify if the reference count of each power well matches its HW enabled
3031  * state and the total refcount of the domains it belongs to. This must be
3032  * called after modeset HW state sanitization, which is responsible for
3033  * acquiring reference counts for any power wells in use and disabling the
3034  * ones left on by BIOS but not required by any active output.
3035  */
3036 void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
3037 {
3038         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3039         struct i915_power_well *power_well;
3040         bool dump_domain_info;
3041
3042         mutex_lock(&power_domains->lock);
3043
3044         dump_domain_info = false;
3045         for_each_power_well(dev_priv, power_well) {
3046                 enum intel_display_power_domain domain;
3047                 int domains_count;
3048                 bool enabled;
3049
3050                 /*
3051                  * Power wells not belonging to any domain (like the MISC_IO
3052                  * and PW1 power wells) are under FW control, so ignore them,
3053                  * since their state can change asynchronously.
3054                  */
3055                 if (!power_well->domains)
3056                         continue;
3057
3058                 enabled = power_well->ops->is_enabled(dev_priv, power_well);
3059                 if ((power_well->count || power_well->always_on) != enabled)
3060                         DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)",
3061                                   power_well->name, power_well->count, enabled);
3062
3063                 domains_count = 0;
3064                 for_each_power_domain(domain, power_well->domains)
3065                         domains_count += power_domains->domain_use_count[domain];
3066
3067                 if (power_well->count != domains_count) {
3068                         DRM_ERROR("power well %s refcount/domain refcount mismatch "
3069                                   "(refcount %d/domains refcount %d)\n",
3070                                   power_well->name, power_well->count,
3071                                   domains_count);
3072                         dump_domain_info = true;
3073                 }
3074         }
3075
3076         if (dump_domain_info) {
3077                 static bool dumped;
3078
3079                 if (!dumped) {
3080                         intel_power_domains_dump_info(dev_priv);
3081                         dumped = true;
3082                 }
3083         }
3084
3085         mutex_unlock(&power_domains->lock);
3086 }
3087
3088 /**
3089  * intel_runtime_pm_get - grab a runtime pm reference
3090  * @dev_priv: i915 device instance
3091  *
3092  * This function grabs a device-level runtime pm reference (mostly used for GEM
3093  * code to ensure the GTT or GT is on) and ensures that it is powered up.
3094  *
3095  * Any runtime pm reference obtained by this function must have a symmetric
3096  * call to intel_runtime_pm_put() to release the reference again.
3097  */
3098 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
3099 {
3100         struct pci_dev *pdev = dev_priv->drm.pdev;
3101         struct device *kdev = &pdev->dev;
3102         int ret;
3103
3104         ret = pm_runtime_get_sync(kdev);
3105         WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3106
3107         atomic_inc(&dev_priv->pm.wakeref_count);
3108         assert_rpm_wakelock_held(dev_priv);
3109 }
3110
3111 /**
3112  * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
3113  * @dev_priv: i915 device instance
3114  *
3115  * This function grabs a device-level runtime pm reference if the device is
3116  * already in use and ensures that it is powered up.
3117  *
3118  * Any runtime pm reference obtained by this function must have a symmetric
3119  * call to intel_runtime_pm_put() to release the reference again.
3120  */
3121 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
3122 {
3123         struct pci_dev *pdev = dev_priv->drm.pdev;
3124         struct device *kdev = &pdev->dev;
3125
3126         if (IS_ENABLED(CONFIG_PM)) {
3127                 int ret = pm_runtime_get_if_in_use(kdev);
3128
3129                 /*
3130                  * In cases runtime PM is disabled by the RPM core and we get
3131                  * an -EINVAL return value we are not supposed to call this
3132                  * function, since the power state is undefined. This applies
3133                  * atm to the late/early system suspend/resume handlers.
3134                  */
3135                 WARN_ONCE(ret < 0,
3136                           "pm_runtime_get_if_in_use() failed: %d\n", ret);
3137                 if (ret <= 0)
3138                         return false;
3139         }
3140
3141         atomic_inc(&dev_priv->pm.wakeref_count);
3142         assert_rpm_wakelock_held(dev_priv);
3143
3144         return true;
3145 }
3146
3147 /**
3148  * intel_runtime_pm_get_noresume - grab a runtime pm reference
3149  * @dev_priv: i915 device instance
3150  *
3151  * This function grabs a device-level runtime pm reference (mostly used for GEM
3152  * code to ensure the GTT or GT is on).
3153  *
3154  * It will _not_ power up the device but instead only check that it's powered
3155  * on.  Therefore it is only valid to call this functions from contexts where
3156  * the device is known to be powered up and where trying to power it up would
3157  * result in hilarity and deadlocks. That pretty much means only the system
3158  * suspend/resume code where this is used to grab runtime pm references for
3159  * delayed setup down in work items.
3160  *
3161  * Any runtime pm reference obtained by this function must have a symmetric
3162  * call to intel_runtime_pm_put() to release the reference again.
3163  */
3164 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
3165 {
3166         struct pci_dev *pdev = dev_priv->drm.pdev;
3167         struct device *kdev = &pdev->dev;
3168
3169         assert_rpm_wakelock_held(dev_priv);
3170         pm_runtime_get_noresume(kdev);
3171
3172         atomic_inc(&dev_priv->pm.wakeref_count);
3173 }
3174
3175 /**
3176  * intel_runtime_pm_put - release a runtime pm reference
3177  * @dev_priv: i915 device instance
3178  *
3179  * This function drops the device-level runtime pm reference obtained by
3180  * intel_runtime_pm_get() and might power down the corresponding
3181  * hardware block right away if this is the last reference.
3182  */
3183 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
3184 {
3185         struct pci_dev *pdev = dev_priv->drm.pdev;
3186         struct device *kdev = &pdev->dev;
3187
3188         assert_rpm_wakelock_held(dev_priv);
3189         atomic_dec(&dev_priv->pm.wakeref_count);
3190
3191         pm_runtime_mark_last_busy(kdev);
3192         pm_runtime_put_autosuspend(kdev);
3193 }
3194
3195 /**
3196  * intel_runtime_pm_enable - enable runtime pm
3197  * @dev_priv: i915 device instance
3198  *
3199  * This function enables runtime pm at the end of the driver load sequence.
3200  *
3201  * Note that this function does currently not enable runtime pm for the
3202  * subordinate display power domains. That is only done on the first modeset
3203  * using intel_display_set_init_power().
3204  */
3205 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
3206 {
3207         struct pci_dev *pdev = dev_priv->drm.pdev;
3208         struct device *kdev = &pdev->dev;
3209
3210         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
3211         pm_runtime_mark_last_busy(kdev);
3212
3213         /*
3214          * Take a permanent reference to disable the RPM functionality and drop
3215          * it only when unloading the driver. Use the low level get/put helpers,
3216          * so the driver's own RPM reference tracking asserts also work on
3217          * platforms without RPM support.
3218          */
3219         if (!HAS_RUNTIME_PM(dev_priv)) {
3220                 int ret;
3221
3222                 pm_runtime_dont_use_autosuspend(kdev);
3223                 ret = pm_runtime_get_sync(kdev);
3224                 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3225         } else {
3226                 pm_runtime_use_autosuspend(kdev);
3227         }
3228
3229         /*
3230          * The core calls the driver load handler with an RPM reference held.
3231          * We drop that here and will reacquire it during unloading in
3232          * intel_power_domains_fini().
3233          */
3234         pm_runtime_put_autosuspend(kdev);
3235 }