drivers/gpu/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *    Daniel Vetter <daniel.vetter@ffwll.ch>
  26  *
  27  */
  28
  29 #include <linux/pm_runtime.h>
  30 #include <linux/vgaarb.h>
  31
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 /**
  36  * DOC: runtime pm
  37  *
  38  * The i915 driver supports dynamic enabling and disabling of entire hardware
  39  * blocks at runtime. This is especially important on the display side where
  40  * software is supposed to control many power gates manually on recent hardware,
  41  * since on the GT side a lot of the power management is done by the hardware.
  42  * But even there some manual control at the device level is required.
  43  *
  44  * Since i915 supports a diverse set of platforms with a unified codebase and
  45  * hardware engineers just love to shuffle functionality around between power
  46  * domains there's a sizeable amount of indirection required. This file provides
  47  * generic functions to the driver for grabbing and releasing references for
  48  * abstract power domains. It then maps those to the actual power wells
  49  * present for a given platform.
  50  */
  51
  52 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  53                                          enum i915_power_well_id power_well_id);
  54
  55 static struct i915_power_well *
  56 lookup_power_well(struct drm_i915_private *dev_priv,
  57                   enum i915_power_well_id power_well_id);
  58
  59 const char *
  60 intel_display_power_domain_str(enum intel_display_power_domain domain)
  61 {
  62         switch (domain) {
  63         case POWER_DOMAIN_PIPE_A:
  64                 return "PIPE_A";
  65         case POWER_DOMAIN_PIPE_B:
  66                 return "PIPE_B";
  67         case POWER_DOMAIN_PIPE_C:
  68                 return "PIPE_C";
  69         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  70                 return "PIPE_A_PANEL_FITTER";
  71         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  72                 return "PIPE_B_PANEL_FITTER";
  73         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  74                 return "PIPE_C_PANEL_FITTER";
  75         case POWER_DOMAIN_TRANSCODER_A:
  76                 return "TRANSCODER_A";
  77         case POWER_DOMAIN_TRANSCODER_B:
  78                 return "TRANSCODER_B";
  79         case POWER_DOMAIN_TRANSCODER_C:
  80                 return "TRANSCODER_C";
  81         case POWER_DOMAIN_TRANSCODER_EDP:
  82                 return "TRANSCODER_EDP";
  83         case POWER_DOMAIN_TRANSCODER_DSI_A:
  84                 return "TRANSCODER_DSI_A";
  85         case POWER_DOMAIN_TRANSCODER_DSI_C:
  86                 return "TRANSCODER_DSI_C";
  87         case POWER_DOMAIN_PORT_DDI_A_LANES:
  88                 return "PORT_DDI_A_LANES";
  89         case POWER_DOMAIN_PORT_DDI_B_LANES:
  90                 return "PORT_DDI_B_LANES";
  91         case POWER_DOMAIN_PORT_DDI_C_LANES:
  92                 return "PORT_DDI_C_LANES";
  93         case POWER_DOMAIN_PORT_DDI_D_LANES:
  94                 return "PORT_DDI_D_LANES";
  95         case POWER_DOMAIN_PORT_DDI_E_LANES:
  96                 return "PORT_DDI_E_LANES";
  97         case POWER_DOMAIN_PORT_DDI_A_IO:
  98                 return "PORT_DDI_A_IO";
  99         case POWER_DOMAIN_PORT_DDI_B_IO:
 100                 return "PORT_DDI_B_IO";
 101         case POWER_DOMAIN_PORT_DDI_C_IO:
 102                 return "PORT_DDI_C_IO";
 103         case POWER_DOMAIN_PORT_DDI_D_IO:
 104                 return "PORT_DDI_D_IO";
 105         case POWER_DOMAIN_PORT_DDI_E_IO:
 106                 return "PORT_DDI_E_IO";
 107         case POWER_DOMAIN_PORT_DSI:
 108                 return "PORT_DSI";
 109         case POWER_DOMAIN_PORT_CRT:
 110                 return "PORT_CRT";
 111         case POWER_DOMAIN_PORT_OTHER:
 112                 return "PORT_OTHER";
 113         case POWER_DOMAIN_VGA:
 114                 return "VGA";
 115         case POWER_DOMAIN_AUDIO:
 116                 return "AUDIO";
 117         case POWER_DOMAIN_PLLS:
 118                 return "PLLS";
 119         case POWER_DOMAIN_AUX_A:
 120                 return "AUX_A";
 121         case POWER_DOMAIN_AUX_B:
 122                 return "AUX_B";
 123         case POWER_DOMAIN_AUX_C:
 124                 return "AUX_C";
 125         case POWER_DOMAIN_AUX_D:
 126                 return "AUX_D";
 127         case POWER_DOMAIN_GMBUS:
 128                 return "GMBUS";
 129         case POWER_DOMAIN_INIT:
 130                 return "INIT";
 131         case POWER_DOMAIN_MODESET:
 132                 return "MODESET";
 133         default:
 134                 MISSING_CASE(domain);
 135                 return "?";
 136         }
 137 }
 138
 139 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 140                                     struct i915_power_well *power_well)
 141 {
 142         DRM_DEBUG_KMS("enabling %s\n", power_well->name);
 143         power_well->ops->enable(dev_priv, power_well);
 144         power_well->hw_enabled = true;
 145 }
 146
 147 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 148                                      struct i915_power_well *power_well)
 149 {
 150         DRM_DEBUG_KMS("disabling %s\n", power_well->name);
 151         power_well->hw_enabled = false;
 152         power_well->ops->disable(dev_priv, power_well);
 153 }
 154
 155 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 156                                  struct i915_power_well *power_well)
 157 {
 158         if (!power_well->count++)
 159                 intel_power_well_enable(dev_priv, power_well);
 160 }
 161
 162 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 163                                  struct i915_power_well *power_well)
 164 {
 165         WARN(!power_well->count, "Use count on power well %s is already zero",
 166              power_well->name);
 167
 168         if (!--power_well->count)
 169                 intel_power_well_disable(dev_priv, power_well);
 170 }
 171
 172 /**
 173  * __intel_display_power_is_enabled - unlocked check for a power domain
 174  * @dev_priv: i915 device instance
 175  * @domain: power domain to check
 176  *
 177  * This is the unlocked version of intel_display_power_is_enabled() and should
 178  * only be used from error capture and recovery code where deadlocks are
 179  * possible.
 180  *
 181  * Returns:
 182  * True when the power domain is enabled, false otherwise.
 183  */
 184 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 185                                       enum intel_display_power_domain domain)
 186 {
 187         struct i915_power_well *power_well;
 188         bool is_enabled;
 189
 190         if (dev_priv->runtime_pm.suspended)
 191                 return false;
 192
 193         is_enabled = true;
 194
 195         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
 196                 if (power_well->always_on)
 197                         continue;
 198
 199                 if (!power_well->hw_enabled) {
 200                         is_enabled = false;
 201                         break;
 202                 }
 203         }
 204
 205         return is_enabled;
 206 }
 207
 208 /**
 209  * intel_display_power_is_enabled - check for a power domain
 210  * @dev_priv: i915 device instance
 211  * @domain: power domain to check
 212  *
 213  * This function can be used to check the hw power domain state. It is mostly
 214  * used in hardware state readout functions. Everywhere else code should rely
 215  * upon explicit power domain reference counting to ensure that the hardware
 216  * block is powered up before accessing it.
 217  *
 218  * Callers must hold the relevant modesetting locks to ensure that concurrent
 219  * threads can't disable the power well while the caller tries to read a few
 220  * registers.
 221  *
 222  * Returns:
 223  * True when the power domain is enabled, false otherwise.
 224  */
 225 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 226                                     enum intel_display_power_domain domain)
 227 {
 228         struct i915_power_domains *power_domains;
 229         bool ret;
 230
 231         power_domains = &dev_priv->power_domains;
 232
 233         mutex_lock(&power_domains->lock);
 234         ret = __intel_display_power_is_enabled(dev_priv, domain);
 235         mutex_unlock(&power_domains->lock);
 236
 237         return ret;
 238 }
 239
 240 /**
 241  * intel_display_set_init_power - set the initial power domain state
 242  * @dev_priv: i915 device instance
 243  * @enable: whether to enable or disable the initial power domain state
 244  *
 245  * For simplicity our driver load/unload and system suspend/resume code assumes
 246  * that all power domains are always enabled. This functions controls the state
 247  * of this little hack. While the initial power domain state is enabled runtime
 248  * pm is effectively disabled.
 249  */
 250 void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 251                                   bool enable)
 252 {
 253         if (dev_priv->power_domains.init_power_on == enable)
 254                 return;
 255
 256         if (enable)
 257                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 258         else
 259                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 260
 261         dev_priv->power_domains.init_power_on = enable;
 262 }
 263
 264 /*
 265  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 266  * when not needed anymore. We have 4 registers that can request the power well
 267  * to be enabled, and it will only be disabled if none of the registers is
 268  * requesting it to be enabled.
 269  */
 270 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 271                                        u8 irq_pipe_mask, bool has_vga)
 272 {
 273         struct pci_dev *pdev = dev_priv->drm.pdev;
 274
 275         /*
 276          * After we re-enable the power well, if we touch VGA register 0x3d5
 277          * we'll get unclaimed register interrupts. This stops after we write
 278          * anything to the VGA MSR register. The vgacon module uses this
 279          * register all the time, so if we unbind our driver and, as a
 280          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 281          * console_unlock(). So make here we touch the VGA MSR register, making
 282          * sure vgacon can keep working normally without triggering interrupts
 283          * and error messages.
 284          */
 285         if (has_vga) {
 286                 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 287                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 288                 vga_put(pdev, VGA_RSRC_LEGACY_IO);
 289         }
 290
 291         if (irq_pipe_mask)
 292                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 293 }
 294
 295 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 296                                        u8 irq_pipe_mask)
 297 {
 298         if (irq_pipe_mask)
 299                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 300 }
 301
 302
 303 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 304                                            struct i915_power_well *power_well)
 305 {
 306         enum i915_power_well_id id = power_well->id;
 307
 308         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 309         WARN_ON(intel_wait_for_register(dev_priv,
 310                                         HSW_PWR_WELL_CTL_DRIVER(id),
 311                                         HSW_PWR_WELL_CTL_STATE(id),
 312                                         HSW_PWR_WELL_CTL_STATE(id),
 313                                         1));
 314 }
 315
 316 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 317                                      enum i915_power_well_id id)
 318 {
 319         u32 req_mask = HSW_PWR_WELL_CTL_REQ(id);
 320         u32 ret;
 321
 322         ret = I915_READ(HSW_PWR_WELL_CTL_BIOS(id)) & req_mask ? 1 : 0;
 323         ret |= I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & req_mask ? 2 : 0;
 324         ret |= I915_READ(HSW_PWR_WELL_CTL_KVMR) & req_mask ? 4 : 0;
 325         ret |= I915_READ(HSW_PWR_WELL_CTL_DEBUG(id)) & req_mask ? 8 : 0;
 326
 327         return ret;
 328 }
 329
 330 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 331                                             struct i915_power_well *power_well)
 332 {
 333         enum i915_power_well_id id = power_well->id;
 334         bool disabled;
 335         u32 reqs;
 336
 337         /*
 338          * Bspec doesn't require waiting for PWs to get disabled, but still do
 339          * this for paranoia. The known cases where a PW will be forced on:
 340          * - a KVMR request on any power well via the KVMR request register
 341          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 342          *   DEBUG request registers
 343          * Skip the wait in case any of the request bits are set and print a
 344          * diagnostic message.
 345          */
 346         wait_for((disabled = !(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 347                                HSW_PWR_WELL_CTL_STATE(id))) ||
 348                  (reqs = hsw_power_well_requesters(dev_priv, id)), 1);
 349         if (disabled)
 350                 return;
 351
 352         DRM_DEBUG_KMS("%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 353                       power_well->name,
 354                       !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 355 }
 356
 357 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 358                                            enum skl_power_gate pg)
 359 {
 360         /* Timeout 5us for PG#0, for other PGs 1us */
 361         WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
 362                                         SKL_FUSE_PG_DIST_STATUS(pg),
 363                                         SKL_FUSE_PG_DIST_STATUS(pg), 1));
 364 }
 365
 366 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 367                                   struct i915_power_well *power_well)
 368 {
 369         enum i915_power_well_id id = power_well->id;
 370         bool wait_fuses = power_well->hsw.has_fuses;
 371         enum skl_power_gate uninitialized_var(pg);
 372         u32 val;
 373
 374         if (wait_fuses) {
 375                 pg = SKL_PW_TO_PG(id);
 376                 /*
 377                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 378                  * before enabling the power well and PW1/PG1's own fuse
 379                  * state after the enabling. For all other power wells with
 380                  * fuses we only have to wait for that PW/PG's fuse state
 381                  * after the enabling.
 382                  */
 383                 if (pg == SKL_PG1)
 384                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 385         }
 386
 387         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 388         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
 389         hsw_wait_for_power_well_enable(dev_priv, power_well);
 390
 391         if (wait_fuses)
 392                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 393
 394         hsw_power_well_post_enable(dev_priv, power_well->hsw.irq_pipe_mask,
 395                                    power_well->hsw.has_vga);
 396 }
 397
 398 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 399                                    struct i915_power_well *power_well)
 400 {
 401         enum i915_power_well_id id = power_well->id;
 402         u32 val;
 403
 404         hsw_power_well_pre_disable(dev_priv, power_well->hsw.irq_pipe_mask);
 405
 406         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 407         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
 408                    val & ~HSW_PWR_WELL_CTL_REQ(id));
 409         hsw_wait_for_power_well_disable(dev_priv, power_well);
 410 }
 411
 412 /*
 413  * We should only use the power well if we explicitly asked the hardware to
 414  * enable it, so check if it's enabled and also check if we've requested it to
 415  * be enabled.
 416  */
 417 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 418                                    struct i915_power_well *power_well)
 419 {
 420         enum i915_power_well_id id = power_well->id;
 421         u32 mask = HSW_PWR_WELL_CTL_REQ(id) | HSW_PWR_WELL_CTL_STATE(id);
 422
 423         return (I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & mask) == mask;
 424 }
 425
 426 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 427 {
 428         enum i915_power_well_id id = SKL_DISP_PW_2;
 429
 430         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 431                   "DC9 already programmed to be enabled.\n");
 432         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 433                   "DC5 still not disabled to enable DC9.\n");
 434         WARN_ONCE(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 435                   HSW_PWR_WELL_CTL_REQ(id),
 436                   "Power well 2 on.\n");
 437         WARN_ONCE(intel_irqs_enabled(dev_priv),
 438                   "Interrupts not disabled yet.\n");
 439
 440          /*
 441           * TODO: check for the following to verify the conditions to enter DC9
 442           * state are satisfied:
 443           * 1] Check relevant display engine registers to verify if mode set
 444           * disable sequence was followed.
 445           * 2] Check if display uninitialize sequence is initialized.
 446           */
 447 }
 448
 449 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 450 {
 451         WARN_ONCE(intel_irqs_enabled(dev_priv),
 452                   "Interrupts not disabled yet.\n");
 453         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 454                   "DC5 still not disabled.\n");
 455
 456          /*
 457           * TODO: check for the following to verify DC9 state was indeed
 458           * entered before programming to disable it:
 459           * 1] Check relevant display engine registers to verify if mode
 460           *  set disable sequence was followed.
 461           * 2] Check if display uninitialize sequence is initialized.
 462           */
 463 }
 464
 465 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 466                                 u32 state)
 467 {
 468         int rewrites = 0;
 469         int rereads = 0;
 470         u32 v;
 471
 472         I915_WRITE(DC_STATE_EN, state);
 473
 474         /* It has been observed that disabling the dc6 state sometimes
 475          * doesn't stick and dmc keeps returning old value. Make sure
 476          * the write really sticks enough times and also force rewrite until
 477          * we are confident that state is exactly what we want.
 478          */
 479         do  {
 480                 v = I915_READ(DC_STATE_EN);
 481
 482                 if (v != state) {
 483                         I915_WRITE(DC_STATE_EN, state);
 484                         rewrites++;
 485                         rereads = 0;
 486                 } else if (rereads++ > 5) {
 487                         break;
 488                 }
 489
 490         } while (rewrites < 100);
 491
 492         if (v != state)
 493                 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
 494                           state, v);
 495
 496         /* Most of the times we need one retry, avoid spam */
 497         if (rewrites > 1)
 498                 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
 499                               state, rewrites);
 500 }
 501
 502 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 503 {
 504         u32 mask;
 505
 506         mask = DC_STATE_EN_UPTO_DC5;
 507         if (IS_GEN9_LP(dev_priv))
 508                 mask |= DC_STATE_EN_DC9;
 509         else
 510                 mask |= DC_STATE_EN_UPTO_DC6;
 511
 512         return mask;
 513 }
 514
 515 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 516 {
 517         u32 val;
 518
 519         val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
 520
 521         DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
 522                       dev_priv->csr.dc_state, val);
 523         dev_priv->csr.dc_state = val;
 524 }
 525
 526 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 527 {
 528         uint32_t val;
 529         uint32_t mask;
 530
 531         if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
 532                 state &= dev_priv->csr.allowed_dc_mask;
 533
 534         val = I915_READ(DC_STATE_EN);
 535         mask = gen9_dc_mask(dev_priv);
 536         DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
 537                       val & mask, state);
 538
 539         /* Check if DMC is ignoring our DC state requests */
 540         if ((val & mask) != dev_priv->csr.dc_state)
 541                 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
 542                           dev_priv->csr.dc_state, val & mask);
 543
 544         val &= ~mask;
 545         val |= state;
 546
 547         gen9_write_dc_state(dev_priv, val);
 548
 549         dev_priv->csr.dc_state = val & mask;
 550 }
 551
 552 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 553 {
 554         assert_can_enable_dc9(dev_priv);
 555
 556         DRM_DEBUG_KMS("Enabling DC9\n");
 557
 558         intel_power_sequencer_reset(dev_priv);
 559         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 560 }
 561
 562 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 563 {
 564         assert_can_disable_dc9(dev_priv);
 565
 566         DRM_DEBUG_KMS("Disabling DC9\n");
 567
 568         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 569
 570         intel_pps_unlock_regs_wa(dev_priv);
 571 }
 572
 573 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 574 {
 575         WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
 576                   "CSR program storage start is NULL\n");
 577         WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
 578         WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 579 }
 580
 581 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 582 {
 583         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 584                                         SKL_DISP_PW_2);
 585
 586         WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 587
 588         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 589                   "DC5 already programmed to be enabled.\n");
 590         assert_rpm_wakelock_held(dev_priv);
 591
 592         assert_csr_loaded(dev_priv);
 593 }
 594
 595 void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 596 {
 597         assert_can_enable_dc5(dev_priv);
 598
 599         DRM_DEBUG_KMS("Enabling DC5\n");
 600
 601         /* Wa Display #1183: skl,kbl,cfl */
 602         if (IS_GEN9_BC(dev_priv))
 603                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 604                            SKL_SELECT_ALTERNATE_DC_EXIT);
 605
 606         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 607 }
 608
 609 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 610 {
 611         WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 612                   "Backlight is not disabled.\n");
 613         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 614                   "DC6 already programmed to be enabled.\n");
 615
 616         assert_csr_loaded(dev_priv);
 617 }
 618
 619 void skl_enable_dc6(struct drm_i915_private *dev_priv)
 620 {
 621         assert_can_enable_dc6(dev_priv);
 622
 623         DRM_DEBUG_KMS("Enabling DC6\n");
 624
 625         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 626
 627 }
 628
 629 void skl_disable_dc6(struct drm_i915_private *dev_priv)
 630 {
 631         DRM_DEBUG_KMS("Disabling DC6\n");
 632
 633         /* Wa Display #1183: skl,kbl,cfl */
 634         if (IS_GEN9_BC(dev_priv))
 635                 I915_WRITE(GEN8_CHICKEN_DCPR_1, I915_READ(GEN8_CHICKEN_DCPR_1) |
 636                            SKL_SELECT_ALTERNATE_DC_EXIT);
 637
 638         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 639 }
 640
 641 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 642                                    struct i915_power_well *power_well)
 643 {
 644         enum i915_power_well_id id = power_well->id;
 645         u32 mask = HSW_PWR_WELL_CTL_REQ(id);
 646         u32 bios_req = I915_READ(HSW_PWR_WELL_CTL_BIOS(id));
 647
 648         /* Take over the request bit if set by BIOS. */
 649         if (bios_req & mask) {
 650                 u32 drv_req = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 651
 652                 if (!(drv_req & mask))
 653                         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), drv_req | mask);
 654                 I915_WRITE(HSW_PWR_WELL_CTL_BIOS(id), bios_req & ~mask);
 655         }
 656 }
 657
 658 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 659                                            struct i915_power_well *power_well)
 660 {
 661         bxt_ddi_phy_init(dev_priv, power_well->bxt.phy);
 662 }
 663
 664 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 665                                             struct i915_power_well *power_well)
 666 {
 667         bxt_ddi_phy_uninit(dev_priv, power_well->bxt.phy);
 668 }
 669
 670 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 671                                             struct i915_power_well *power_well)
 672 {
 673         return bxt_ddi_phy_is_enabled(dev_priv, power_well->bxt.phy);
 674 }
 675
 676 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 677 {
 678         struct i915_power_well *power_well;
 679
 680         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 681         if (power_well->count > 0)
 682                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 683
 684         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 685         if (power_well->count > 0)
 686                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 687
 688         if (IS_GEMINILAKE(dev_priv)) {
 689                 power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
 690                 if (power_well->count > 0)
 691                         bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 692         }
 693 }
 694
 695 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 696                                            struct i915_power_well *power_well)
 697 {
 698         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 699 }
 700
 701 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 702 {
 703         u32 tmp = I915_READ(DBUF_CTL);
 704
 705         WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
 706              (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
 707              "Unexpected DBuf power power state (0x%08x)\n", tmp);
 708 }
 709
 710 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 711                                           struct i915_power_well *power_well)
 712 {
 713         struct intel_cdclk_state cdclk_state = {};
 714
 715         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 716
 717         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
 718         WARN_ON(!intel_cdclk_state_compare(&dev_priv->cdclk.hw, &cdclk_state));
 719
 720         gen9_assert_dbuf_enabled(dev_priv);
 721
 722         if (IS_GEN9_LP(dev_priv))
 723                 bxt_verify_ddi_phy_power_wells(dev_priv);
 724 }
 725
 726 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 727                                            struct i915_power_well *power_well)
 728 {
 729         if (!dev_priv->csr.dmc_payload)
 730                 return;
 731
 732         if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 733                 skl_enable_dc6(dev_priv);
 734         else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 735                 gen9_enable_dc5(dev_priv);
 736 }
 737
 738 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
 739                                          struct i915_power_well *power_well)
 740 {
 741 }
 742
 743 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 744                                            struct i915_power_well *power_well)
 745 {
 746 }
 747
 748 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 749                                              struct i915_power_well *power_well)
 750 {
 751         return true;
 752 }
 753
 754 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
 755                                          struct i915_power_well *power_well)
 756 {
 757         if ((I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
 758                 i830_enable_pipe(dev_priv, PIPE_A);
 759         if ((I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
 760                 i830_enable_pipe(dev_priv, PIPE_B);
 761 }
 762
 763 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
 764                                           struct i915_power_well *power_well)
 765 {
 766         i830_disable_pipe(dev_priv, PIPE_B);
 767         i830_disable_pipe(dev_priv, PIPE_A);
 768 }
 769
 770 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
 771                                           struct i915_power_well *power_well)
 772 {
 773         return I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
 774                 I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 775 }
 776
 777 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
 778                                           struct i915_power_well *power_well)
 779 {
 780         if (power_well->count > 0)
 781                 i830_pipes_power_well_enable(dev_priv, power_well);
 782         else
 783                 i830_pipes_power_well_disable(dev_priv, power_well);
 784 }
 785
 786 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 787                                struct i915_power_well *power_well, bool enable)
 788 {
 789         enum i915_power_well_id power_well_id = power_well->id;
 790         u32 mask;
 791         u32 state;
 792         u32 ctrl;
 793
 794         mask = PUNIT_PWRGT_MASK(power_well_id);
 795         state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 796                          PUNIT_PWRGT_PWR_GATE(power_well_id);
 797
 798         mutex_lock(&dev_priv->pcu_lock);
 799
 800 #define COND \
 801         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 802
 803         if (COND)
 804                 goto out;
 805
 806         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 807         ctrl &= ~mask;
 808         ctrl |= state;
 809         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 810
 811         if (wait_for(COND, 100))
 812                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 813                           state,
 814                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 815
 816 #undef COND
 817
 818 out:
 819         mutex_unlock(&dev_priv->pcu_lock);
 820 }
 821
 822 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 823                                   struct i915_power_well *power_well)
 824 {
 825         vlv_set_power_well(dev_priv, power_well, true);
 826 }
 827
 828 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 829                                    struct i915_power_well *power_well)
 830 {
 831         vlv_set_power_well(dev_priv, power_well, false);
 832 }
 833
 834 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 835                                    struct i915_power_well *power_well)
 836 {
 837         enum i915_power_well_id power_well_id = power_well->id;
 838         bool enabled = false;
 839         u32 mask;
 840         u32 state;
 841         u32 ctrl;
 842
 843         mask = PUNIT_PWRGT_MASK(power_well_id);
 844         ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
 845
 846         mutex_lock(&dev_priv->pcu_lock);
 847
 848         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
 849         /*
 850          * We only ever set the power-on and power-gate states, anything
 851          * else is unexpected.
 852          */
 853         WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
 854                 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
 855         if (state == ctrl)
 856                 enabled = true;
 857
 858         /*
 859          * A transient state at this point would mean some unexpected party
 860          * is poking at the power controls too.
 861          */
 862         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 863         WARN_ON(ctrl != state);
 864
 865         mutex_unlock(&dev_priv->pcu_lock);
 866
 867         return enabled;
 868 }
 869
 870 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 871 {
 872         u32 val;
 873
 874         /*
 875          * On driver load, a pipe may be active and driving a DSI display.
 876          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
 877          * (and never recovering) in this case. intel_dsi_post_disable() will
 878          * clear it when we turn off the display.
 879          */
 880         val = I915_READ(DSPCLK_GATE_D);
 881         val &= DPOUNIT_CLOCK_GATE_DISABLE;
 882         val |= VRHUNIT_CLOCK_GATE_DISABLE;
 883         I915_WRITE(DSPCLK_GATE_D, val);
 884
 885         /*
 886          * Disable trickle feed and enable pnd deadline calculation
 887          */
 888         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 889         I915_WRITE(CBR1_VLV, 0);
 890
 891         WARN_ON(dev_priv->rawclk_freq == 0);
 892
 893         I915_WRITE(RAWCLK_FREQ_VLV,
 894                    DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
 895 }
 896
 897 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 898 {
 899         struct intel_encoder *encoder;
 900         enum pipe pipe;
 901
 902         /*
 903          * Enable the CRI clock source so we can get at the
 904          * display and the reference clock for VGA
 905          * hotplug / manual detection. Supposedly DSI also
 906          * needs the ref clock up and running.
 907          *
 908          * CHV DPLL B/C have some issues if VGA mode is enabled.
 909          */
 910         for_each_pipe(dev_priv, pipe) {
 911                 u32 val = I915_READ(DPLL(pipe));
 912
 913                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
 914                 if (pipe != PIPE_A)
 915                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
 916
 917                 I915_WRITE(DPLL(pipe), val);
 918         }
 919
 920         vlv_init_display_clock_gating(dev_priv);
 921
 922         spin_lock_irq(&dev_priv->irq_lock);
 923         valleyview_enable_display_irqs(dev_priv);
 924         spin_unlock_irq(&dev_priv->irq_lock);
 925
 926         /*
 927          * During driver initialization/resume we can avoid restoring the
 928          * part of the HW/SW state that will be inited anyway explicitly.
 929          */
 930         if (dev_priv->power_domains.initializing)
 931                 return;
 932
 933         intel_hpd_init(dev_priv);
 934
 935         /* Re-enable the ADPA, if we have one */
 936         for_each_intel_encoder(&dev_priv->drm, encoder) {
 937                 if (encoder->type == INTEL_OUTPUT_ANALOG)
 938                         intel_crt_reset(&encoder->base);
 939         }
 940
 941         i915_redisable_vga_power_on(dev_priv);
 942
 943         intel_pps_unlock_regs_wa(dev_priv);
 944 }
 945
 946 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
 947 {
 948         spin_lock_irq(&dev_priv->irq_lock);
 949         valleyview_disable_display_irqs(dev_priv);
 950         spin_unlock_irq(&dev_priv->irq_lock);
 951
 952         /* make sure we're done processing display irqs */
 953         synchronize_irq(dev_priv->drm.irq);
 954
 955         intel_power_sequencer_reset(dev_priv);
 956
 957         /* Prevent us from re-enabling polling on accident in late suspend */
 958         if (!dev_priv->drm.dev->power.is_suspended)
 959                 intel_hpd_poll_init(dev_priv);
 960 }
 961
 962 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 963                                           struct i915_power_well *power_well)
 964 {
 965         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 966
 967         vlv_set_power_well(dev_priv, power_well, true);
 968
 969         vlv_display_power_well_init(dev_priv);
 970 }
 971
 972 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 973                                            struct i915_power_well *power_well)
 974 {
 975         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 976
 977         vlv_display_power_well_deinit(dev_priv);
 978
 979         vlv_set_power_well(dev_priv, power_well, false);
 980 }
 981
 982 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 983                                            struct i915_power_well *power_well)
 984 {
 985         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
 986
 987         /* since ref/cri clock was enabled */
 988         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 989
 990         vlv_set_power_well(dev_priv, power_well, true);
 991
 992         /*
 993          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
 994          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
 995          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
 996          *   b. The other bits such as sfr settings / modesel may all
 997          *      be set to 0.
 998          *
 999          * This should only be done on init and resume from S3 with
1000          * both PLLs disabled, or we risk losing DPIO and PLL
1001          * synchronization.
1002          */
1003         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
1004 }
1005
1006 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1007                                             struct i915_power_well *power_well)
1008 {
1009         enum pipe pipe;
1010
1011         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1012
1013         for_each_pipe(dev_priv, pipe)
1014                 assert_pll_disabled(dev_priv, pipe);
1015
1016         /* Assert common reset */
1017         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1018
1019         vlv_set_power_well(dev_priv, power_well, false);
1020 }
1021
1022 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1023
1024 static struct i915_power_well *
1025 lookup_power_well(struct drm_i915_private *dev_priv,
1026                   enum i915_power_well_id power_well_id)
1027 {
1028         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1029         int i;
1030
1031         for (i = 0; i < power_domains->power_well_count; i++) {
1032                 struct i915_power_well *power_well;
1033
1034                 power_well = &power_domains->power_wells[i];
1035                 if (power_well->id == power_well_id)
1036                         return power_well;
1037         }
1038
1039         return NULL;
1040 }
1041
1042 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1043
1044 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1045 {
1046         struct i915_power_well *cmn_bc =
1047                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1048         struct i915_power_well *cmn_d =
1049                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1050         u32 phy_control = dev_priv->chv_phy_control;
1051         u32 phy_status = 0;
1052         u32 phy_status_mask = 0xffffffff;
1053
1054         /*
1055          * The BIOS can leave the PHY is some weird state
1056          * where it doesn't fully power down some parts.
1057          * Disable the asserts until the PHY has been fully
1058          * reset (ie. the power well has been disabled at
1059          * least once).
1060          */
1061         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1062                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1063                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1064                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1065                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1066                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1067                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1068
1069         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1070                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1071                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1072                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1073
1074         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
1075                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1076
1077                 /* this assumes override is only used to enable lanes */
1078                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1079                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1080
1081                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1082                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1083
1084                 /* CL1 is on whenever anything is on in either channel */
1085                 if (BITS_SET(phy_control,
1086                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1087                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1088                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1089
1090                 /*
1091                  * The DPLLB check accounts for the pipe B + port A usage
1092                  * with CL2 powered up but all the lanes in the second channel
1093                  * powered down.
1094                  */
1095                 if (BITS_SET(phy_control,
1096                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1097                     (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1098                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1099
1100                 if (BITS_SET(phy_control,
1101                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1102                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1103                 if (BITS_SET(phy_control,
1104                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1105                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1106
1107                 if (BITS_SET(phy_control,
1108                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1109                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1110                 if (BITS_SET(phy_control,
1111                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1112                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1113         }
1114
1115         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
1116                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1117
1118                 /* this assumes override is only used to enable lanes */
1119                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1120                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1121
1122                 if (BITS_SET(phy_control,
1123                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1124                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1125
1126                 if (BITS_SET(phy_control,
1127                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1128                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1129                 if (BITS_SET(phy_control,
1130                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1131                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1132         }
1133
1134         phy_status &= phy_status_mask;
1135
1136         /*
1137          * The PHY may be busy with some initial calibration and whatnot,
1138          * so the power state can take a while to actually change.
1139          */
1140         if (intel_wait_for_register(dev_priv,
1141                                     DISPLAY_PHY_STATUS,
1142                                     phy_status_mask,
1143                                     phy_status,
1144                                     10))
1145                 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1146                           I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1147                            phy_status, dev_priv->chv_phy_control);
1148 }
1149
1150 #undef BITS_SET
1151
1152 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1153                                            struct i915_power_well *power_well)
1154 {
1155         enum dpio_phy phy;
1156         enum pipe pipe;
1157         uint32_t tmp;
1158
1159         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1160                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1161
1162         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1163                 pipe = PIPE_A;
1164                 phy = DPIO_PHY0;
1165         } else {
1166                 pipe = PIPE_C;
1167                 phy = DPIO_PHY1;
1168         }
1169
1170         /* since ref/cri clock was enabled */
1171         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1172         vlv_set_power_well(dev_priv, power_well, true);
1173
1174         /* Poll for phypwrgood signal */
1175         if (intel_wait_for_register(dev_priv,
1176                                     DISPLAY_PHY_STATUS,
1177                                     PHY_POWERGOOD(phy),
1178                                     PHY_POWERGOOD(phy),
1179                                     1))
1180                 DRM_ERROR("Display PHY %d is not power up\n", phy);
1181
1182         mutex_lock(&dev_priv->sb_lock);
1183
1184         /* Enable dynamic power down */
1185         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1186         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1187                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1188         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1189
1190         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1191                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1192                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1193                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1194         } else {
1195                 /*
1196                  * Force the non-existing CL2 off. BXT does this
1197                  * too, so maybe it saves some power even though
1198                  * CL2 doesn't exist?
1199                  */
1200                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1201                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1202                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1203         }
1204
1205         mutex_unlock(&dev_priv->sb_lock);
1206
1207         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1208         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1209
1210         DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1211                       phy, dev_priv->chv_phy_control);
1212
1213         assert_chv_phy_status(dev_priv);
1214 }
1215
1216 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1217                                             struct i915_power_well *power_well)
1218 {
1219         enum dpio_phy phy;
1220
1221         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1222                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1223
1224         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1225                 phy = DPIO_PHY0;
1226                 assert_pll_disabled(dev_priv, PIPE_A);
1227                 assert_pll_disabled(dev_priv, PIPE_B);
1228         } else {
1229                 phy = DPIO_PHY1;
1230                 assert_pll_disabled(dev_priv, PIPE_C);
1231         }
1232
1233         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1234         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1235
1236         vlv_set_power_well(dev_priv, power_well, false);
1237
1238         DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1239                       phy, dev_priv->chv_phy_control);
1240
1241         /* PHY is fully reset now, so we can enable the PHY state asserts */
1242         dev_priv->chv_phy_assert[phy] = true;
1243
1244         assert_chv_phy_status(dev_priv);
1245 }
1246
1247 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1248                                      enum dpio_channel ch, bool override, unsigned int mask)
1249 {
1250         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1251         u32 reg, val, expected, actual;
1252
1253         /*
1254          * The BIOS can leave the PHY is some weird state
1255          * where it doesn't fully power down some parts.
1256          * Disable the asserts until the PHY has been fully
1257          * reset (ie. the power well has been disabled at
1258          * least once).
1259          */
1260         if (!dev_priv->chv_phy_assert[phy])
1261                 return;
1262
1263         if (ch == DPIO_CH0)
1264                 reg = _CHV_CMN_DW0_CH0;
1265         else
1266                 reg = _CHV_CMN_DW6_CH1;
1267
1268         mutex_lock(&dev_priv->sb_lock);
1269         val = vlv_dpio_read(dev_priv, pipe, reg);
1270         mutex_unlock(&dev_priv->sb_lock);
1271
1272         /*
1273          * This assumes !override is only used when the port is disabled.
1274          * All lanes should power down even without the override when
1275          * the port is disabled.
1276          */
1277         if (!override || mask == 0xf) {
1278                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1279                 /*
1280                  * If CH1 common lane is not active anymore
1281                  * (eg. for pipe B DPLL) the entire channel will
1282                  * shut down, which causes the common lane registers
1283                  * to read as 0. That means we can't actually check
1284                  * the lane power down status bits, but as the entire
1285                  * register reads as 0 it's a good indication that the
1286                  * channel is indeed entirely powered down.
1287                  */
1288                 if (ch == DPIO_CH1 && val == 0)
1289                         expected = 0;
1290         } else if (mask != 0x0) {
1291                 expected = DPIO_ANYDL_POWERDOWN;
1292         } else {
1293                 expected = 0;
1294         }
1295
1296         if (ch == DPIO_CH0)
1297                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1298         else
1299                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1300         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1301
1302         WARN(actual != expected,
1303              "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1304              !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1305              !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1306              reg, val);
1307 }
1308
1309 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1310                           enum dpio_channel ch, bool override)
1311 {
1312         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1313         bool was_override;
1314
1315         mutex_lock(&power_domains->lock);
1316
1317         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1318
1319         if (override == was_override)
1320                 goto out;
1321
1322         if (override)
1323                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1324         else
1325                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1326
1327         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1328
1329         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1330                       phy, ch, dev_priv->chv_phy_control);
1331
1332         assert_chv_phy_status(dev_priv);
1333
1334 out:
1335         mutex_unlock(&power_domains->lock);
1336
1337         return was_override;
1338 }
1339
1340 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1341                              bool override, unsigned int mask)
1342 {
1343         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1344         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1345         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1346         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1347
1348         mutex_lock(&power_domains->lock);
1349
1350         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1351         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1352
1353         if (override)
1354                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1355         else
1356                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1357
1358         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1359
1360         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1361                       phy, ch, mask, dev_priv->chv_phy_control);
1362
1363         assert_chv_phy_status(dev_priv);
1364
1365         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1366
1367         mutex_unlock(&power_domains->lock);
1368 }
1369
1370 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1371                                         struct i915_power_well *power_well)
1372 {
1373         enum pipe pipe = PIPE_A;
1374         bool enabled;
1375         u32 state, ctrl;
1376
1377         mutex_lock(&dev_priv->pcu_lock);
1378
1379         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1380         /*
1381          * We only ever set the power-on and power-gate states, anything
1382          * else is unexpected.
1383          */
1384         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1385         enabled = state == DP_SSS_PWR_ON(pipe);
1386
1387         /*
1388          * A transient state at this point would mean some unexpected party
1389          * is poking at the power controls too.
1390          */
1391         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1392         WARN_ON(ctrl << 16 != state);
1393
1394         mutex_unlock(&dev_priv->pcu_lock);
1395
1396         return enabled;
1397 }
1398
1399 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1400                                     struct i915_power_well *power_well,
1401                                     bool enable)
1402 {
1403         enum pipe pipe = PIPE_A;
1404         u32 state;
1405         u32 ctrl;
1406
1407         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1408
1409         mutex_lock(&dev_priv->pcu_lock);
1410
1411 #define COND \
1412         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1413
1414         if (COND)
1415                 goto out;
1416
1417         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1418         ctrl &= ~DP_SSC_MASK(pipe);
1419         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1420         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1421
1422         if (wait_for(COND, 100))
1423                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1424                           state,
1425                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1426
1427 #undef COND
1428
1429 out:
1430         mutex_unlock(&dev_priv->pcu_lock);
1431 }
1432
1433 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1434                                        struct i915_power_well *power_well)
1435 {
1436         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1437
1438         chv_set_pipe_power_well(dev_priv, power_well, true);
1439
1440         vlv_display_power_well_init(dev_priv);
1441 }
1442
1443 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1444                                         struct i915_power_well *power_well)
1445 {
1446         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1447
1448         vlv_display_power_well_deinit(dev_priv);
1449
1450         chv_set_pipe_power_well(dev_priv, power_well, false);
1451 }
1452
1453 static void
1454 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1455                                  enum intel_display_power_domain domain)
1456 {
1457         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1458         struct i915_power_well *power_well;
1459
1460         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
1461                 intel_power_well_get(dev_priv, power_well);
1462
1463         power_domains->domain_use_count[domain]++;
1464 }
1465
1466 /**
1467  * intel_display_power_get - grab a power domain reference
1468  * @dev_priv: i915 device instance
1469  * @domain: power domain to reference
1470  *
1471  * This function grabs a power domain reference for @domain and ensures that the
1472  * power domain and all its parents are powered up. Therefore users should only
1473  * grab a reference to the innermost power domain they need.
1474  *
1475  * Any power domain reference obtained by this function must have a symmetric
1476  * call to intel_display_power_put() to release the reference again.
1477  */
1478 void intel_display_power_get(struct drm_i915_private *dev_priv,
1479                              enum intel_display_power_domain domain)
1480 {
1481         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1482
1483         intel_runtime_pm_get(dev_priv);
1484
1485         mutex_lock(&power_domains->lock);
1486
1487         __intel_display_power_get_domain(dev_priv, domain);
1488
1489         mutex_unlock(&power_domains->lock);
1490 }
1491
1492 /**
1493  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1494  * @dev_priv: i915 device instance
1495  * @domain: power domain to reference
1496  *
1497  * This function grabs a power domain reference for @domain and ensures that the
1498  * power domain and all its parents are powered up. Therefore users should only
1499  * grab a reference to the innermost power domain they need.
1500  *
1501  * Any power domain reference obtained by this function must have a symmetric
1502  * call to intel_display_power_put() to release the reference again.
1503  */
1504 bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1505                                         enum intel_display_power_domain domain)
1506 {
1507         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1508         bool is_enabled;
1509
1510         if (!intel_runtime_pm_get_if_in_use(dev_priv))
1511                 return false;
1512
1513         mutex_lock(&power_domains->lock);
1514
1515         if (__intel_display_power_is_enabled(dev_priv, domain)) {
1516                 __intel_display_power_get_domain(dev_priv, domain);
1517                 is_enabled = true;
1518         } else {
1519                 is_enabled = false;
1520         }
1521
1522         mutex_unlock(&power_domains->lock);
1523
1524         if (!is_enabled)
1525                 intel_runtime_pm_put(dev_priv);
1526
1527         return is_enabled;
1528 }
1529
1530 /**
1531  * intel_display_power_put - release a power domain reference
1532  * @dev_priv: i915 device instance
1533  * @domain: power domain to reference
1534  *
1535  * This function drops the power domain reference obtained by
1536  * intel_display_power_get() and might power down the corresponding hardware
1537  * block right away if this is the last reference.
1538  */
1539 void intel_display_power_put(struct drm_i915_private *dev_priv,
1540                              enum intel_display_power_domain domain)
1541 {
1542         struct i915_power_domains *power_domains;
1543         struct i915_power_well *power_well;
1544
1545         power_domains = &dev_priv->power_domains;
1546
1547         mutex_lock(&power_domains->lock);
1548
1549         WARN(!power_domains->domain_use_count[domain],
1550              "Use count on domain %s is already zero\n",
1551              intel_display_power_domain_str(domain));
1552         power_domains->domain_use_count[domain]--;
1553
1554         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
1555                 intel_power_well_put(dev_priv, power_well);
1556
1557         mutex_unlock(&power_domains->lock);
1558
1559         intel_runtime_pm_put(dev_priv);
1560 }
1561
1562 #define I830_PIPES_POWER_DOMAINS (              \
1563         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1564         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1565         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1566         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1567         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1568         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1569         BIT_ULL(POWER_DOMAIN_INIT))
1570
1571 #define VLV_DISPLAY_POWER_DOMAINS (             \
1572         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1573         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1574         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1575         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1576         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1577         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1578         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1579         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1580         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1581         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1582         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1583         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1584         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1585         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1586         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1587         BIT_ULL(POWER_DOMAIN_INIT))
1588
1589 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1590         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1591         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1592         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1593         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1594         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1595         BIT_ULL(POWER_DOMAIN_INIT))
1596
1597 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1598         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1599         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1600         BIT_ULL(POWER_DOMAIN_INIT))
1601
1602 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1603         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1604         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1605         BIT_ULL(POWER_DOMAIN_INIT))
1606
1607 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1608         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1609         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1610         BIT_ULL(POWER_DOMAIN_INIT))
1611
1612 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1613         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1614         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1615         BIT_ULL(POWER_DOMAIN_INIT))
1616
1617 #define CHV_DISPLAY_POWER_DOMAINS (             \
1618         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1619         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1620         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
1621         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1622         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1623         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1624         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1625         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1626         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
1627         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1628         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1629         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1630         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1631         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1632         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1633         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1634         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1635         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1636         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1637         BIT_ULL(POWER_DOMAIN_INIT))
1638
1639 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1640         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1641         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1642         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1643         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1644         BIT_ULL(POWER_DOMAIN_INIT))
1645
1646 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1647         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1648         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1649         BIT_ULL(POWER_DOMAIN_INIT))
1650
1651 #define HSW_DISPLAY_POWER_DOMAINS (                     \
1652         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1653         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1654         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
1655         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1656         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1657         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1658         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1659         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1660         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1661         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1662         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1663         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1664         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1665         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1666         BIT_ULL(POWER_DOMAIN_INIT))
1667
1668 #define BDW_DISPLAY_POWER_DOMAINS (                     \
1669         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1670         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1671         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1672         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1673         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1674         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1675         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1676         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1677         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1678         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1679         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1680         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1681         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1682         BIT_ULL(POWER_DOMAIN_INIT))
1683
1684 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1685         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1686         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1687         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1688         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1689         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1690         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1691         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1692         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1693         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1694         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1695         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
1696         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1697         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1698         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1699         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1700         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1701         BIT_ULL(POWER_DOMAIN_INIT))
1702 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
1703         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1704         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
1705         BIT_ULL(POWER_DOMAIN_INIT))
1706 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1707         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1708         BIT_ULL(POWER_DOMAIN_INIT))
1709 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1710         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1711         BIT_ULL(POWER_DOMAIN_INIT))
1712 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
1713         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1714         BIT_ULL(POWER_DOMAIN_INIT))
1715 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1716         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1717         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1718         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1719         BIT_ULL(POWER_DOMAIN_INIT))
1720
1721 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1722         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1723         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1724         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1725         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1726         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1727         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1728         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1729         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1730         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1731         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1732         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1733         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1734         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1735         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1736         BIT_ULL(POWER_DOMAIN_INIT))
1737 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1738         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1739         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1740         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1741         BIT_ULL(POWER_DOMAIN_INIT))
1742 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
1743         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1744         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1745         BIT_ULL(POWER_DOMAIN_INIT))
1746 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
1747         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1748         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1749         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1750         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1751         BIT_ULL(POWER_DOMAIN_INIT))
1752
1753 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1754         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1755         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1756         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1757         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1758         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1759         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1760         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1761         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1762         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1763         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1764         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1765         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1766         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1767         BIT_ULL(POWER_DOMAIN_INIT))
1768 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
1769         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
1770 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1771         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
1772 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1773         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
1774 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
1775         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1776         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1777         BIT_ULL(POWER_DOMAIN_INIT))
1778 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
1779         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1780         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1781         BIT_ULL(POWER_DOMAIN_INIT))
1782 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
1783         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1784         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1785         BIT_ULL(POWER_DOMAIN_INIT))
1786 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
1787         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
1788         BIT_ULL(POWER_DOMAIN_INIT))
1789 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
1790         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1791         BIT_ULL(POWER_DOMAIN_INIT))
1792 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
1793         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1794         BIT_ULL(POWER_DOMAIN_INIT))
1795 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1796         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1797         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1798         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1799         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1800         BIT_ULL(POWER_DOMAIN_INIT))
1801
1802 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1803         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1804         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1805         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1806         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1807         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1808         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1809         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1810         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1811         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1812         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1813         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1814         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1815         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1816         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1817         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1818         BIT_ULL(POWER_DOMAIN_INIT))
1819 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
1820         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1821         BIT_ULL(POWER_DOMAIN_INIT))
1822 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
1823         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1824         BIT_ULL(POWER_DOMAIN_INIT))
1825 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
1826         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1827         BIT_ULL(POWER_DOMAIN_INIT))
1828 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
1829         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1830         BIT_ULL(POWER_DOMAIN_INIT))
1831 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
1832         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1833         BIT_ULL(POWER_DOMAIN_INIT))
1834 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
1835         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1836         BIT_ULL(POWER_DOMAIN_INIT))
1837 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
1838         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1839         BIT_ULL(POWER_DOMAIN_INIT))
1840 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
1841         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1842         BIT_ULL(POWER_DOMAIN_INIT))
1843 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1844         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1845         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1846         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1847         BIT_ULL(POWER_DOMAIN_INIT))
1848
1849 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1850         .sync_hw = i9xx_power_well_sync_hw_noop,
1851         .enable = i9xx_always_on_power_well_noop,
1852         .disable = i9xx_always_on_power_well_noop,
1853         .is_enabled = i9xx_always_on_power_well_enabled,
1854 };
1855
1856 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
1857         .sync_hw = i9xx_power_well_sync_hw_noop,
1858         .enable = chv_pipe_power_well_enable,
1859         .disable = chv_pipe_power_well_disable,
1860         .is_enabled = chv_pipe_power_well_enabled,
1861 };
1862
1863 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1864         .sync_hw = i9xx_power_well_sync_hw_noop,
1865         .enable = chv_dpio_cmn_power_well_enable,
1866         .disable = chv_dpio_cmn_power_well_disable,
1867         .is_enabled = vlv_power_well_enabled,
1868 };
1869
1870 static struct i915_power_well i9xx_always_on_power_well[] = {
1871         {
1872                 .name = "always-on",
1873                 .always_on = 1,
1874                 .domains = POWER_DOMAIN_MASK,
1875                 .ops = &i9xx_always_on_power_well_ops,
1876                 .id = I915_DISP_PW_ALWAYS_ON,
1877         },
1878 };
1879
1880 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
1881         .sync_hw = i830_pipes_power_well_sync_hw,
1882         .enable = i830_pipes_power_well_enable,
1883         .disable = i830_pipes_power_well_disable,
1884         .is_enabled = i830_pipes_power_well_enabled,
1885 };
1886
1887 static struct i915_power_well i830_power_wells[] = {
1888         {
1889                 .name = "always-on",
1890                 .always_on = 1,
1891                 .domains = POWER_DOMAIN_MASK,
1892                 .ops = &i9xx_always_on_power_well_ops,
1893                 .id = I915_DISP_PW_ALWAYS_ON,
1894         },
1895         {
1896                 .name = "pipes",
1897                 .domains = I830_PIPES_POWER_DOMAINS,
1898                 .ops = &i830_pipes_power_well_ops,
1899                 .id = I830_DISP_PW_PIPES,
1900         },
1901 };
1902
1903 static const struct i915_power_well_ops hsw_power_well_ops = {
1904         .sync_hw = hsw_power_well_sync_hw,
1905         .enable = hsw_power_well_enable,
1906         .disable = hsw_power_well_disable,
1907         .is_enabled = hsw_power_well_enabled,
1908 };
1909
1910 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1911         .sync_hw = i9xx_power_well_sync_hw_noop,
1912         .enable = gen9_dc_off_power_well_enable,
1913         .disable = gen9_dc_off_power_well_disable,
1914         .is_enabled = gen9_dc_off_power_well_enabled,
1915 };
1916
1917 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1918         .sync_hw = i9xx_power_well_sync_hw_noop,
1919         .enable = bxt_dpio_cmn_power_well_enable,
1920         .disable = bxt_dpio_cmn_power_well_disable,
1921         .is_enabled = bxt_dpio_cmn_power_well_enabled,
1922 };
1923
1924 static struct i915_power_well hsw_power_wells[] = {
1925         {
1926                 .name = "always-on",
1927                 .always_on = 1,
1928                 .domains = POWER_DOMAIN_MASK,
1929                 .ops = &i9xx_always_on_power_well_ops,
1930                 .id = I915_DISP_PW_ALWAYS_ON,
1931         },
1932         {
1933                 .name = "display",
1934                 .domains = HSW_DISPLAY_POWER_DOMAINS,
1935                 .ops = &hsw_power_well_ops,
1936                 .id = HSW_DISP_PW_GLOBAL,
1937                 {
1938                         .hsw.has_vga = true,
1939                 },
1940         },
1941 };
1942
1943 static struct i915_power_well bdw_power_wells[] = {
1944         {
1945                 .name = "always-on",
1946                 .always_on = 1,
1947                 .domains = POWER_DOMAIN_MASK,
1948                 .ops = &i9xx_always_on_power_well_ops,
1949                 .id = I915_DISP_PW_ALWAYS_ON,
1950         },
1951         {
1952                 .name = "display",
1953                 .domains = BDW_DISPLAY_POWER_DOMAINS,
1954                 .ops = &hsw_power_well_ops,
1955                 .id = HSW_DISP_PW_GLOBAL,
1956                 {
1957                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
1958                         .hsw.has_vga = true,
1959                 },
1960         },
1961 };
1962
1963 static const struct i915_power_well_ops vlv_display_power_well_ops = {
1964         .sync_hw = i9xx_power_well_sync_hw_noop,
1965         .enable = vlv_display_power_well_enable,
1966         .disable = vlv_display_power_well_disable,
1967         .is_enabled = vlv_power_well_enabled,
1968 };
1969
1970 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1971         .sync_hw = i9xx_power_well_sync_hw_noop,
1972         .enable = vlv_dpio_cmn_power_well_enable,
1973         .disable = vlv_dpio_cmn_power_well_disable,
1974         .is_enabled = vlv_power_well_enabled,
1975 };
1976
1977 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1978         .sync_hw = i9xx_power_well_sync_hw_noop,
1979         .enable = vlv_power_well_enable,
1980         .disable = vlv_power_well_disable,
1981         .is_enabled = vlv_power_well_enabled,
1982 };
1983
1984 static struct i915_power_well vlv_power_wells[] = {
1985         {
1986                 .name = "always-on",
1987                 .always_on = 1,
1988                 .domains = POWER_DOMAIN_MASK,
1989                 .ops = &i9xx_always_on_power_well_ops,
1990                 .id = I915_DISP_PW_ALWAYS_ON,
1991         },
1992         {
1993                 .name = "display",
1994                 .domains = VLV_DISPLAY_POWER_DOMAINS,
1995                 .id = PUNIT_POWER_WELL_DISP2D,
1996                 .ops = &vlv_display_power_well_ops,
1997         },
1998         {
1999                 .name = "dpio-tx-b-01",
2000                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2001                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2002                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2003                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2004                 .ops = &vlv_dpio_power_well_ops,
2005                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
2006         },
2007         {
2008                 .name = "dpio-tx-b-23",
2009                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2010                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2011                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2012                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2013                 .ops = &vlv_dpio_power_well_ops,
2014                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
2015         },
2016         {
2017                 .name = "dpio-tx-c-01",
2018                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2019                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2020                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2021                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2022                 .ops = &vlv_dpio_power_well_ops,
2023                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
2024         },
2025         {
2026                 .name = "dpio-tx-c-23",
2027                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2028                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2029                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2030                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2031                 .ops = &vlv_dpio_power_well_ops,
2032                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
2033         },
2034         {
2035                 .name = "dpio-common",
2036                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2037                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2038                 .ops = &vlv_dpio_cmn_power_well_ops,
2039         },
2040 };
2041
2042 static struct i915_power_well chv_power_wells[] = {
2043         {
2044                 .name = "always-on",
2045                 .always_on = 1,
2046                 .domains = POWER_DOMAIN_MASK,
2047                 .ops = &i9xx_always_on_power_well_ops,
2048                 .id = I915_DISP_PW_ALWAYS_ON,
2049         },
2050         {
2051                 .name = "display",
2052                 /*
2053                  * Pipe A power well is the new disp2d well. Pipe B and C
2054                  * power wells don't actually exist. Pipe A power well is
2055                  * required for any pipe to work.
2056                  */
2057                 .domains = CHV_DISPLAY_POWER_DOMAINS,
2058                 .id = CHV_DISP_PW_PIPE_A,
2059                 .ops = &chv_pipe_power_well_ops,
2060         },
2061         {
2062                 .name = "dpio-common-bc",
2063                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2064                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2065                 .ops = &chv_dpio_cmn_power_well_ops,
2066         },
2067         {
2068                 .name = "dpio-common-d",
2069                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2070                 .id = PUNIT_POWER_WELL_DPIO_CMN_D,
2071                 .ops = &chv_dpio_cmn_power_well_ops,
2072         },
2073 };
2074
2075 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2076                                          enum i915_power_well_id power_well_id)
2077 {
2078         struct i915_power_well *power_well;
2079         bool ret;
2080
2081         power_well = lookup_power_well(dev_priv, power_well_id);
2082         ret = power_well->ops->is_enabled(dev_priv, power_well);
2083
2084         return ret;
2085 }
2086
2087 static struct i915_power_well skl_power_wells[] = {
2088         {
2089                 .name = "always-on",
2090                 .always_on = 1,
2091                 .domains = POWER_DOMAIN_MASK,
2092                 .ops = &i9xx_always_on_power_well_ops,
2093                 .id = I915_DISP_PW_ALWAYS_ON,
2094         },
2095         {
2096                 .name = "power well 1",
2097                 /* Handled by the DMC firmware */
2098                 .domains = 0,
2099                 .ops = &hsw_power_well_ops,
2100                 .id = SKL_DISP_PW_1,
2101                 {
2102                         .hsw.has_fuses = true,
2103                 },
2104         },
2105         {
2106                 .name = "MISC IO power well",
2107                 /* Handled by the DMC firmware */
2108                 .domains = 0,
2109                 .ops = &hsw_power_well_ops,
2110                 .id = SKL_DISP_PW_MISC_IO,
2111         },
2112         {
2113                 .name = "DC off",
2114                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2115                 .ops = &gen9_dc_off_power_well_ops,
2116                 .id = SKL_DISP_PW_DC_OFF,
2117         },
2118         {
2119                 .name = "power well 2",
2120                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2121                 .ops = &hsw_power_well_ops,
2122                 .id = SKL_DISP_PW_2,
2123                 {
2124                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2125                         .hsw.has_vga = true,
2126                         .hsw.has_fuses = true,
2127                 },
2128         },
2129         {
2130                 .name = "DDI A/E IO power well",
2131                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
2132                 .ops = &hsw_power_well_ops,
2133                 .id = SKL_DISP_PW_DDI_A_E,
2134         },
2135         {
2136                 .name = "DDI B IO power well",
2137                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2138                 .ops = &hsw_power_well_ops,
2139                 .id = SKL_DISP_PW_DDI_B,
2140         },
2141         {
2142                 .name = "DDI C IO power well",
2143                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2144                 .ops = &hsw_power_well_ops,
2145                 .id = SKL_DISP_PW_DDI_C,
2146         },
2147         {
2148                 .name = "DDI D IO power well",
2149                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
2150                 .ops = &hsw_power_well_ops,
2151                 .id = SKL_DISP_PW_DDI_D,
2152         },
2153 };
2154
2155 static struct i915_power_well bxt_power_wells[] = {
2156         {
2157                 .name = "always-on",
2158                 .always_on = 1,
2159                 .domains = POWER_DOMAIN_MASK,
2160                 .ops = &i9xx_always_on_power_well_ops,
2161                 .id = I915_DISP_PW_ALWAYS_ON,
2162         },
2163         {
2164                 .name = "power well 1",
2165                 .domains = 0,
2166                 .ops = &hsw_power_well_ops,
2167                 .id = SKL_DISP_PW_1,
2168                 {
2169                         .hsw.has_fuses = true,
2170                 },
2171         },
2172         {
2173                 .name = "DC off",
2174                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2175                 .ops = &gen9_dc_off_power_well_ops,
2176                 .id = SKL_DISP_PW_DC_OFF,
2177         },
2178         {
2179                 .name = "power well 2",
2180                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2181                 .ops = &hsw_power_well_ops,
2182                 .id = SKL_DISP_PW_2,
2183                 {
2184                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2185                         .hsw.has_vga = true,
2186                         .hsw.has_fuses = true,
2187                 },
2188         },
2189         {
2190                 .name = "dpio-common-a",
2191                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2192                 .ops = &bxt_dpio_cmn_power_well_ops,
2193                 .id = BXT_DPIO_CMN_A,
2194                 {
2195                         .bxt.phy = DPIO_PHY1,
2196                 },
2197         },
2198         {
2199                 .name = "dpio-common-bc",
2200                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2201                 .ops = &bxt_dpio_cmn_power_well_ops,
2202                 .id = BXT_DPIO_CMN_BC,
2203                 {
2204                         .bxt.phy = DPIO_PHY0,
2205                 },
2206         },
2207 };
2208
2209 static struct i915_power_well glk_power_wells[] = {
2210         {
2211                 .name = "always-on",
2212                 .always_on = 1,
2213                 .domains = POWER_DOMAIN_MASK,
2214                 .ops = &i9xx_always_on_power_well_ops,
2215                 .id = I915_DISP_PW_ALWAYS_ON,
2216         },
2217         {
2218                 .name = "power well 1",
2219                 /* Handled by the DMC firmware */
2220                 .domains = 0,
2221                 .ops = &hsw_power_well_ops,
2222                 .id = SKL_DISP_PW_1,
2223                 {
2224                         .hsw.has_fuses = true,
2225                 },
2226         },
2227         {
2228                 .name = "DC off",
2229                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2230                 .ops = &gen9_dc_off_power_well_ops,
2231                 .id = SKL_DISP_PW_DC_OFF,
2232         },
2233         {
2234                 .name = "power well 2",
2235                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2236                 .ops = &hsw_power_well_ops,
2237                 .id = SKL_DISP_PW_2,
2238                 {
2239                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2240                         .hsw.has_vga = true,
2241                         .hsw.has_fuses = true,
2242                 },
2243         },
2244         {
2245                 .name = "dpio-common-a",
2246                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2247                 .ops = &bxt_dpio_cmn_power_well_ops,
2248                 .id = BXT_DPIO_CMN_A,
2249                 {
2250                         .bxt.phy = DPIO_PHY1,
2251                 },
2252         },
2253         {
2254                 .name = "dpio-common-b",
2255                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2256                 .ops = &bxt_dpio_cmn_power_well_ops,
2257                 .id = BXT_DPIO_CMN_BC,
2258                 {
2259                         .bxt.phy = DPIO_PHY0,
2260                 },
2261         },
2262         {
2263                 .name = "dpio-common-c",
2264                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2265                 .ops = &bxt_dpio_cmn_power_well_ops,
2266                 .id = GLK_DPIO_CMN_C,
2267                 {
2268                         .bxt.phy = DPIO_PHY2,
2269                 },
2270         },
2271         {
2272                 .name = "AUX A",
2273                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2274                 .ops = &hsw_power_well_ops,
2275                 .id = GLK_DISP_PW_AUX_A,
2276         },
2277         {
2278                 .name = "AUX B",
2279                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2280                 .ops = &hsw_power_well_ops,
2281                 .id = GLK_DISP_PW_AUX_B,
2282         },
2283         {
2284                 .name = "AUX C",
2285                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2286                 .ops = &hsw_power_well_ops,
2287                 .id = GLK_DISP_PW_AUX_C,
2288         },
2289         {
2290                 .name = "DDI A IO power well",
2291                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
2292                 .ops = &hsw_power_well_ops,
2293                 .id = GLK_DISP_PW_DDI_A,
2294         },
2295         {
2296                 .name = "DDI B IO power well",
2297                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2298                 .ops = &hsw_power_well_ops,
2299                 .id = SKL_DISP_PW_DDI_B,
2300         },
2301         {
2302                 .name = "DDI C IO power well",
2303                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2304                 .ops = &hsw_power_well_ops,
2305                 .id = SKL_DISP_PW_DDI_C,
2306         },
2307 };
2308
2309 static struct i915_power_well cnl_power_wells[] = {
2310         {
2311                 .name = "always-on",
2312                 .always_on = 1,
2313                 .domains = POWER_DOMAIN_MASK,
2314                 .ops = &i9xx_always_on_power_well_ops,
2315                 .id = I915_DISP_PW_ALWAYS_ON,
2316         },
2317         {
2318                 .name = "power well 1",
2319                 /* Handled by the DMC firmware */
2320                 .domains = 0,
2321                 .ops = &hsw_power_well_ops,
2322                 .id = SKL_DISP_PW_1,
2323                 {
2324                         .hsw.has_fuses = true,
2325                 },
2326         },
2327         {
2328                 .name = "AUX A",
2329                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
2330                 .ops = &hsw_power_well_ops,
2331                 .id = CNL_DISP_PW_AUX_A,
2332         },
2333         {
2334                 .name = "AUX B",
2335                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
2336                 .ops = &hsw_power_well_ops,
2337                 .id = CNL_DISP_PW_AUX_B,
2338         },
2339         {
2340                 .name = "AUX C",
2341                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
2342                 .ops = &hsw_power_well_ops,
2343                 .id = CNL_DISP_PW_AUX_C,
2344         },
2345         {
2346                 .name = "AUX D",
2347                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
2348                 .ops = &hsw_power_well_ops,
2349                 .id = CNL_DISP_PW_AUX_D,
2350         },
2351         {
2352                 .name = "DC off",
2353                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
2354                 .ops = &gen9_dc_off_power_well_ops,
2355                 .id = SKL_DISP_PW_DC_OFF,
2356         },
2357         {
2358                 .name = "power well 2",
2359                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2360                 .ops = &hsw_power_well_ops,
2361                 .id = SKL_DISP_PW_2,
2362                 {
2363                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2364                         .hsw.has_vga = true,
2365                         .hsw.has_fuses = true,
2366                 },
2367         },
2368         {
2369                 .name = "DDI A IO power well",
2370                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
2371                 .ops = &hsw_power_well_ops,
2372                 .id = CNL_DISP_PW_DDI_A,
2373         },
2374         {
2375                 .name = "DDI B IO power well",
2376                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
2377                 .ops = &hsw_power_well_ops,
2378                 .id = SKL_DISP_PW_DDI_B,
2379         },
2380         {
2381                 .name = "DDI C IO power well",
2382                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
2383                 .ops = &hsw_power_well_ops,
2384                 .id = SKL_DISP_PW_DDI_C,
2385         },
2386         {
2387                 .name = "DDI D IO power well",
2388                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
2389                 .ops = &hsw_power_well_ops,
2390                 .id = SKL_DISP_PW_DDI_D,
2391         },
2392 };
2393
2394 static int
2395 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2396                                    int disable_power_well)
2397 {
2398         if (disable_power_well >= 0)
2399                 return !!disable_power_well;
2400
2401         return 1;
2402 }
2403
2404 static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2405                                     int enable_dc)
2406 {
2407         uint32_t mask;
2408         int requested_dc;
2409         int max_dc;
2410
2411         if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
2412                 max_dc = 2;
2413                 mask = 0;
2414         } else if (IS_GEN9_LP(dev_priv)) {
2415                 max_dc = 1;
2416                 /*
2417                  * DC9 has a separate HW flow from the rest of the DC states,
2418                  * not depending on the DMC firmware. It's needed by system
2419                  * suspend/resume, so allow it unconditionally.
2420                  */
2421                 mask = DC_STATE_EN_DC9;
2422         } else {
2423                 max_dc = 0;
2424                 mask = 0;
2425         }
2426
2427         if (!i915_modparams.disable_power_well)
2428                 max_dc = 0;
2429
2430         if (enable_dc >= 0 && enable_dc <= max_dc) {
2431                 requested_dc = enable_dc;
2432         } else if (enable_dc == -1) {
2433                 requested_dc = max_dc;
2434         } else if (enable_dc > max_dc && enable_dc <= 2) {
2435                 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
2436                               enable_dc, max_dc);
2437                 requested_dc = max_dc;
2438         } else {
2439                 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
2440                 requested_dc = max_dc;
2441         }
2442
2443         if (requested_dc > 1)
2444                 mask |= DC_STATE_EN_UPTO_DC6;
2445         if (requested_dc > 0)
2446                 mask |= DC_STATE_EN_UPTO_DC5;
2447
2448         DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
2449
2450         return mask;
2451 }
2452
2453 static void assert_power_well_ids_unique(struct drm_i915_private *dev_priv)
2454 {
2455         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2456         u64 power_well_ids;
2457         int i;
2458
2459         power_well_ids = 0;
2460         for (i = 0; i < power_domains->power_well_count; i++) {
2461                 enum i915_power_well_id id = power_domains->power_wells[i].id;
2462
2463                 WARN_ON(id >= sizeof(power_well_ids) * 8);
2464                 WARN_ON(power_well_ids & BIT_ULL(id));
2465                 power_well_ids |= BIT_ULL(id);
2466         }
2467 }
2468
2469 #define set_power_wells(power_domains, __power_wells) ({                \
2470         (power_domains)->power_wells = (__power_wells);                 \
2471         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
2472 })
2473
2474 /**
2475  * intel_power_domains_init - initializes the power domain structures
2476  * @dev_priv: i915 device instance
2477  *
2478  * Initializes the power domain structures for @dev_priv depending upon the
2479  * supported platform.
2480  */
2481 int intel_power_domains_init(struct drm_i915_private *dev_priv)
2482 {
2483         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2484
2485         i915_modparams.disable_power_well =
2486                 sanitize_disable_power_well_option(dev_priv,
2487                                                    i915_modparams.disable_power_well);
2488         dev_priv->csr.allowed_dc_mask =
2489                 get_allowed_dc_mask(dev_priv, i915_modparams.enable_dc);
2490
2491         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
2492
2493         mutex_init(&power_domains->lock);
2494
2495         /*
2496          * The enabling order will be from lower to higher indexed wells,
2497          * the disabling order is reversed.
2498          */
2499         if (IS_HASWELL(dev_priv)) {
2500                 set_power_wells(power_domains, hsw_power_wells);
2501         } else if (IS_BROADWELL(dev_priv)) {
2502                 set_power_wells(power_domains, bdw_power_wells);
2503         } else if (IS_GEN9_BC(dev_priv)) {
2504                 set_power_wells(power_domains, skl_power_wells);
2505         } else if (IS_CANNONLAKE(dev_priv)) {
2506                 set_power_wells(power_domains, cnl_power_wells);
2507         } else if (IS_BROXTON(dev_priv)) {
2508                 set_power_wells(power_domains, bxt_power_wells);
2509         } else if (IS_GEMINILAKE(dev_priv)) {
2510                 set_power_wells(power_domains, glk_power_wells);
2511         } else if (IS_CHERRYVIEW(dev_priv)) {
2512                 set_power_wells(power_domains, chv_power_wells);
2513         } else if (IS_VALLEYVIEW(dev_priv)) {
2514                 set_power_wells(power_domains, vlv_power_wells);
2515         } else if (IS_I830(dev_priv)) {
2516                 set_power_wells(power_domains, i830_power_wells);
2517         } else {
2518                 set_power_wells(power_domains, i9xx_always_on_power_well);
2519         }
2520
2521         assert_power_well_ids_unique(dev_priv);
2522
2523         return 0;
2524 }
2525
2526 /**
2527  * intel_power_domains_fini - finalizes the power domain structures
2528  * @dev_priv: i915 device instance
2529  *
2530  * Finalizes the power domain structures for @dev_priv depending upon the
2531  * supported platform. This function also disables runtime pm and ensures that
2532  * the device stays powered up so that the driver can be reloaded.
2533  */
2534 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
2535 {
2536         struct device *kdev = &dev_priv->drm.pdev->dev;
2537
2538         /*
2539          * The i915.ko module is still not prepared to be loaded when
2540          * the power well is not enabled, so just enable it in case
2541          * we're going to unload/reload.
2542          * The following also reacquires the RPM reference the core passed
2543          * to the driver during loading, which is dropped in
2544          * intel_runtime_pm_enable(). We have to hand back the control of the
2545          * device to the core with this reference held.
2546          */
2547         intel_display_set_init_power(dev_priv, true);
2548
2549         /* Remove the refcount we took to keep power well support disabled. */
2550         if (!i915_modparams.disable_power_well)
2551                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2552
2553         /*
2554          * Remove the refcount we took in intel_runtime_pm_enable() in case
2555          * the platform doesn't support runtime PM.
2556          */
2557         if (!HAS_RUNTIME_PM(dev_priv))
2558                 pm_runtime_put(kdev);
2559 }
2560
2561 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
2562 {
2563         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2564         struct i915_power_well *power_well;
2565
2566         mutex_lock(&power_domains->lock);
2567         for_each_power_well(dev_priv, power_well) {
2568                 power_well->ops->sync_hw(dev_priv, power_well);
2569                 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
2570                                                                      power_well);
2571         }
2572         mutex_unlock(&power_domains->lock);
2573 }
2574
2575 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
2576 {
2577         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
2578         POSTING_READ(DBUF_CTL);
2579
2580         udelay(10);
2581
2582         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
2583                 DRM_ERROR("DBuf power enable timeout\n");
2584 }
2585
2586 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
2587 {
2588         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
2589         POSTING_READ(DBUF_CTL);
2590
2591         udelay(10);
2592
2593         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
2594                 DRM_ERROR("DBuf power disable timeout!\n");
2595 }
2596
2597 static void skl_display_core_init(struct drm_i915_private *dev_priv,
2598                                    bool resume)
2599 {
2600         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2601         struct i915_power_well *well;
2602         uint32_t val;
2603
2604         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2605
2606         /* enable PCH reset handshake */
2607         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2608         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
2609
2610         /* enable PG1 and Misc I/O */
2611         mutex_lock(&power_domains->lock);
2612
2613         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2614         intel_power_well_enable(dev_priv, well);
2615
2616         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2617         intel_power_well_enable(dev_priv, well);
2618
2619         mutex_unlock(&power_domains->lock);
2620
2621         skl_init_cdclk(dev_priv);
2622
2623         gen9_dbuf_enable(dev_priv);
2624
2625         if (resume && dev_priv->csr.dmc_payload)
2626                 intel_csr_load_program(dev_priv);
2627 }
2628
2629 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
2630 {
2631         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2632         struct i915_power_well *well;
2633
2634         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2635
2636         gen9_dbuf_disable(dev_priv);
2637
2638         skl_uninit_cdclk(dev_priv);
2639
2640         /* The spec doesn't call for removing the reset handshake flag */
2641         /* disable PG1 and Misc I/O */
2642
2643         mutex_lock(&power_domains->lock);
2644
2645         /*
2646          * BSpec says to keep the MISC IO power well enabled here, only
2647          * remove our request for power well 1.
2648          * Note that even though the driver's request is removed power well 1
2649          * may stay enabled after this due to DMC's own request on it.
2650          */
2651         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2652         intel_power_well_disable(dev_priv, well);
2653
2654         mutex_unlock(&power_domains->lock);
2655
2656         usleep_range(10, 30);           /* 10 us delay per Bspec */
2657 }
2658
2659 void bxt_display_core_init(struct drm_i915_private *dev_priv,
2660                            bool resume)
2661 {
2662         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2663         struct i915_power_well *well;
2664         uint32_t val;
2665
2666         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2667
2668         /*
2669          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
2670          * or else the reset will hang because there is no PCH to respond.
2671          * Move the handshake programming to initialization sequence.
2672          * Previously was left up to BIOS.
2673          */
2674         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2675         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
2676         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2677
2678         /* Enable PG1 */
2679         mutex_lock(&power_domains->lock);
2680
2681         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2682         intel_power_well_enable(dev_priv, well);
2683
2684         mutex_unlock(&power_domains->lock);
2685
2686         bxt_init_cdclk(dev_priv);
2687
2688         gen9_dbuf_enable(dev_priv);
2689
2690         if (resume && dev_priv->csr.dmc_payload)
2691                 intel_csr_load_program(dev_priv);
2692 }
2693
2694 void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
2695 {
2696         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2697         struct i915_power_well *well;
2698
2699         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2700
2701         gen9_dbuf_disable(dev_priv);
2702
2703         bxt_uninit_cdclk(dev_priv);
2704
2705         /* The spec doesn't call for removing the reset handshake flag */
2706
2707         /*
2708          * Disable PW1 (PG1).
2709          * Note that even though the driver's request is removed power well 1
2710          * may stay enabled after this due to DMC's own request on it.
2711          */
2712         mutex_lock(&power_domains->lock);
2713
2714         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2715         intel_power_well_disable(dev_priv, well);
2716
2717         mutex_unlock(&power_domains->lock);
2718
2719         usleep_range(10, 30);           /* 10 us delay per Bspec */
2720 }
2721
2722 enum {
2723         PROCMON_0_85V_DOT_0,
2724         PROCMON_0_95V_DOT_0,
2725         PROCMON_0_95V_DOT_1,
2726         PROCMON_1_05V_DOT_0,
2727         PROCMON_1_05V_DOT_1,
2728 };
2729
2730 static const struct cnl_procmon {
2731         u32 dw1, dw9, dw10;
2732 } cnl_procmon_values[] = {
2733         [PROCMON_0_85V_DOT_0] =
2734                 { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, },
2735         [PROCMON_0_95V_DOT_0] =
2736                 { .dw1 = 0x00000000, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, },
2737         [PROCMON_0_95V_DOT_1] =
2738                 { .dw1 = 0x00000000, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, },
2739         [PROCMON_1_05V_DOT_0] =
2740                 { .dw1 = 0x00000000, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, },
2741         [PROCMON_1_05V_DOT_1] =
2742                 { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, },
2743 };
2744
2745 static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv)
2746 {
2747         const struct cnl_procmon *procmon;
2748         u32 val;
2749
2750         val = I915_READ(CNL_PORT_COMP_DW3);
2751         switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) {
2752         default:
2753                 MISSING_CASE(val);
2754         case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0:
2755                 procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0];
2756                 break;
2757         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0:
2758                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0];
2759                 break;
2760         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1:
2761                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1];
2762                 break;
2763         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0:
2764                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0];
2765                 break;
2766         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1:
2767                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1];
2768                 break;
2769         }
2770
2771         val = I915_READ(CNL_PORT_COMP_DW1);
2772         val &= ~((0xff << 16) | 0xff);
2773         val |= procmon->dw1;
2774         I915_WRITE(CNL_PORT_COMP_DW1, val);
2775
2776         I915_WRITE(CNL_PORT_COMP_DW9, procmon->dw9);
2777         I915_WRITE(CNL_PORT_COMP_DW10, procmon->dw10);
2778 }
2779
2780 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
2781 {
2782         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2783         struct i915_power_well *well;
2784         u32 val;
2785
2786         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2787
2788         /* 1. Enable PCH Reset Handshake */
2789         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2790         val |= RESET_PCH_HANDSHAKE_ENABLE;
2791         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2792
2793         /* 2. Enable Comp */
2794         val = I915_READ(CHICKEN_MISC_2);
2795         val &= ~CNL_COMP_PWR_DOWN;
2796         I915_WRITE(CHICKEN_MISC_2, val);
2797
2798         cnl_set_procmon_ref_values(dev_priv);
2799
2800         val = I915_READ(CNL_PORT_COMP_DW0);
2801         val |= COMP_INIT;
2802         I915_WRITE(CNL_PORT_COMP_DW0, val);
2803
2804         /* 3. */
2805         val = I915_READ(CNL_PORT_CL1CM_DW5);
2806         val |= CL_POWER_DOWN_ENABLE;
2807         I915_WRITE(CNL_PORT_CL1CM_DW5, val);
2808
2809         /*
2810          * 4. Enable Power Well 1 (PG1).
2811          *    The AUX IO power wells will be enabled on demand.
2812          */
2813         mutex_lock(&power_domains->lock);
2814         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2815         intel_power_well_enable(dev_priv, well);
2816         mutex_unlock(&power_domains->lock);
2817
2818         /* 5. Enable CD clock */
2819         cnl_init_cdclk(dev_priv);
2820
2821         /* 6. Enable DBUF */
2822         gen9_dbuf_enable(dev_priv);
2823
2824         if (resume && dev_priv->csr.dmc_payload)
2825                 intel_csr_load_program(dev_priv);
2826 }
2827
2828 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
2829 {
2830         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2831         struct i915_power_well *well;
2832         u32 val;
2833
2834         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2835
2836         /* 1. Disable all display engine functions -> aready done */
2837
2838         /* 2. Disable DBUF */
2839         gen9_dbuf_disable(dev_priv);
2840
2841         /* 3. Disable CD clock */
2842         cnl_uninit_cdclk(dev_priv);
2843
2844         /*
2845          * 4. Disable Power Well 1 (PG1).
2846          *    The AUX IO power wells are toggled on demand, so they are already
2847          *    disabled at this point.
2848          */
2849         mutex_lock(&power_domains->lock);
2850         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2851         intel_power_well_disable(dev_priv, well);
2852         mutex_unlock(&power_domains->lock);
2853
2854         usleep_range(10, 30);           /* 10 us delay per Bspec */
2855
2856         /* 5. Disable Comp */
2857         val = I915_READ(CHICKEN_MISC_2);
2858         val |= CNL_COMP_PWR_DOWN;
2859         I915_WRITE(CHICKEN_MISC_2, val);
2860 }
2861
2862 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
2863 {
2864         struct i915_power_well *cmn_bc =
2865                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2866         struct i915_power_well *cmn_d =
2867                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
2868
2869         /*
2870          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
2871          * workaround never ever read DISPLAY_PHY_CONTROL, and
2872          * instead maintain a shadow copy ourselves. Use the actual
2873          * power well state and lane status to reconstruct the
2874          * expected initial value.
2875          */
2876         dev_priv->chv_phy_control =
2877                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
2878                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
2879                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
2880                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
2881                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
2882
2883         /*
2884          * If all lanes are disabled we leave the override disabled
2885          * with all power down bits cleared to match the state we
2886          * would use after disabling the port. Otherwise enable the
2887          * override and set the lane powerdown bits accding to the
2888          * current lane status.
2889          */
2890         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
2891                 uint32_t status = I915_READ(DPLL(PIPE_A));
2892                 unsigned int mask;
2893
2894                 mask = status & DPLL_PORTB_READY_MASK;
2895                 if (mask == 0xf)
2896                         mask = 0x0;
2897                 else
2898                         dev_priv->chv_phy_control |=
2899                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
2900
2901                 dev_priv->chv_phy_control |=
2902                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
2903
2904                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
2905                 if (mask == 0xf)
2906                         mask = 0x0;
2907                 else
2908                         dev_priv->chv_phy_control |=
2909                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
2910
2911                 dev_priv->chv_phy_control |=
2912                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
2913
2914                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
2915
2916                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
2917         } else {
2918                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
2919         }
2920
2921         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
2922                 uint32_t status = I915_READ(DPIO_PHY_STATUS);
2923                 unsigned int mask;
2924
2925                 mask = status & DPLL_PORTD_READY_MASK;
2926
2927                 if (mask == 0xf)
2928                         mask = 0x0;
2929                 else
2930                         dev_priv->chv_phy_control |=
2931                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
2932
2933                 dev_priv->chv_phy_control |=
2934                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
2935
2936                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
2937
2938                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
2939         } else {
2940                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
2941         }
2942
2943         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
2944
2945         DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
2946                       dev_priv->chv_phy_control);
2947 }
2948
2949 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
2950 {
2951         struct i915_power_well *cmn =
2952                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2953         struct i915_power_well *disp2d =
2954                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
2955
2956         /* If the display might be already active skip this */
2957         if (cmn->ops->is_enabled(dev_priv, cmn) &&
2958             disp2d->ops->is_enabled(dev_priv, disp2d) &&
2959             I915_READ(DPIO_CTL) & DPIO_CMNRST)
2960                 return;
2961
2962         DRM_DEBUG_KMS("toggling display PHY side reset\n");
2963
2964         /* cmnlane needs DPLL registers */
2965         disp2d->ops->enable(dev_priv, disp2d);
2966
2967         /*
2968          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
2969          * Need to assert and de-assert PHY SB reset by gating the
2970          * common lane power, then un-gating it.
2971          * Simply ungating isn't enough to reset the PHY enough to get
2972          * ports and lanes running.
2973          */
2974         cmn->ops->disable(dev_priv, cmn);
2975 }
2976
2977 /**
2978  * intel_power_domains_init_hw - initialize hardware power domain state
2979  * @dev_priv: i915 device instance
2980  * @resume: Called from resume code paths or not
2981  *
2982  * This function initializes the hardware power domain state and enables all
2983  * power wells belonging to the INIT power domain. Power wells in other
2984  * domains (and not in the INIT domain) are referenced or disabled during the
2985  * modeset state HW readout. After that the reference count of each power well
2986  * must match its HW enabled state, see intel_power_domains_verify_state().
2987  */
2988 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
2989 {
2990         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2991
2992         power_domains->initializing = true;
2993
2994         if (IS_CANNONLAKE(dev_priv)) {
2995                 cnl_display_core_init(dev_priv, resume);
2996         } else if (IS_GEN9_BC(dev_priv)) {
2997                 skl_display_core_init(dev_priv, resume);
2998         } else if (IS_GEN9_LP(dev_priv)) {
2999                 bxt_display_core_init(dev_priv, resume);
3000         } else if (IS_CHERRYVIEW(dev_priv)) {
3001                 mutex_lock(&power_domains->lock);
3002                 chv_phy_control_init(dev_priv);
3003                 mutex_unlock(&power_domains->lock);
3004         } else if (IS_VALLEYVIEW(dev_priv)) {
3005                 mutex_lock(&power_domains->lock);
3006                 vlv_cmnlane_wa(dev_priv);
3007                 mutex_unlock(&power_domains->lock);
3008         }
3009
3010         /* For now, we need the power well to be always enabled. */
3011         intel_display_set_init_power(dev_priv, true);
3012         /* Disable power support if the user asked so. */
3013         if (!i915_modparams.disable_power_well)
3014                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
3015         intel_power_domains_sync_hw(dev_priv);
3016         power_domains->initializing = false;
3017 }
3018
3019 /**
3020  * intel_power_domains_suspend - suspend power domain state
3021  * @dev_priv: i915 device instance
3022  *
3023  * This function prepares the hardware power domain state before entering
3024  * system suspend. It must be paired with intel_power_domains_init_hw().
3025  */
3026 void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
3027 {
3028         /*
3029          * Even if power well support was disabled we still want to disable
3030          * power wells while we are system suspended.
3031          */
3032         if (!i915_modparams.disable_power_well)
3033                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3034
3035         if (IS_CANNONLAKE(dev_priv))
3036                 cnl_display_core_uninit(dev_priv);
3037         else if (IS_GEN9_BC(dev_priv))
3038                 skl_display_core_uninit(dev_priv);
3039         else if (IS_GEN9_LP(dev_priv))
3040                 bxt_display_core_uninit(dev_priv);
3041 }
3042
3043 static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv)
3044 {
3045         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3046         struct i915_power_well *power_well;
3047
3048         for_each_power_well(dev_priv, power_well) {
3049                 enum intel_display_power_domain domain;
3050
3051                 DRM_DEBUG_DRIVER("%-25s %d\n",
3052                                  power_well->name, power_well->count);
3053
3054                 for_each_power_domain(domain, power_well->domains)
3055                         DRM_DEBUG_DRIVER("  %-23s %d\n",
3056                                          intel_display_power_domain_str(domain),
3057                                          power_domains->domain_use_count[domain]);
3058         }
3059 }
3060
3061 /**
3062  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
3063  * @dev_priv: i915 device instance
3064  *
3065  * Verify if the reference count of each power well matches its HW enabled
3066  * state and the total refcount of the domains it belongs to. This must be
3067  * called after modeset HW state sanitization, which is responsible for
3068  * acquiring reference counts for any power wells in use and disabling the
3069  * ones left on by BIOS but not required by any active output.
3070  */
3071 void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
3072 {
3073         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3074         struct i915_power_well *power_well;
3075         bool dump_domain_info;
3076
3077         mutex_lock(&power_domains->lock);
3078
3079         dump_domain_info = false;
3080         for_each_power_well(dev_priv, power_well) {
3081                 enum intel_display_power_domain domain;
3082                 int domains_count;
3083                 bool enabled;
3084
3085                 /*
3086                  * Power wells not belonging to any domain (like the MISC_IO
3087                  * and PW1 power wells) are under FW control, so ignore them,
3088                  * since their state can change asynchronously.
3089                  */
3090                 if (!power_well->domains)
3091                         continue;
3092
3093                 enabled = power_well->ops->is_enabled(dev_priv, power_well);
3094                 if ((power_well->count || power_well->always_on) != enabled)
3095                         DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)",
3096                                   power_well->name, power_well->count, enabled);
3097
3098                 domains_count = 0;
3099                 for_each_power_domain(domain, power_well->domains)
3100                         domains_count += power_domains->domain_use_count[domain];
3101
3102                 if (power_well->count != domains_count) {
3103                         DRM_ERROR("power well %s refcount/domain refcount mismatch "
3104                                   "(refcount %d/domains refcount %d)\n",
3105                                   power_well->name, power_well->count,
3106                                   domains_count);
3107                         dump_domain_info = true;
3108                 }
3109         }
3110
3111         if (dump_domain_info) {
3112                 static bool dumped;
3113
3114                 if (!dumped) {
3115                         intel_power_domains_dump_info(dev_priv);
3116                         dumped = true;
3117                 }
3118         }
3119
3120         mutex_unlock(&power_domains->lock);
3121 }
3122
3123 /**
3124  * intel_runtime_pm_get - grab a runtime pm reference
3125  * @dev_priv: i915 device instance
3126  *
3127  * This function grabs a device-level runtime pm reference (mostly used for GEM
3128  * code to ensure the GTT or GT is on) and ensures that it is powered up.
3129  *
3130  * Any runtime pm reference obtained by this function must have a symmetric
3131  * call to intel_runtime_pm_put() to release the reference again.
3132  */
3133 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
3134 {
3135         struct pci_dev *pdev = dev_priv->drm.pdev;
3136         struct device *kdev = &pdev->dev;
3137         int ret;
3138
3139         ret = pm_runtime_get_sync(kdev);
3140         WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3141
3142         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3143         assert_rpm_wakelock_held(dev_priv);
3144 }
3145
3146 /**
3147  * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
3148  * @dev_priv: i915 device instance
3149  *
3150  * This function grabs a device-level runtime pm reference if the device is
3151  * already in use and ensures that it is powered up.
3152  *
3153  * Any runtime pm reference obtained by this function must have a symmetric
3154  * call to intel_runtime_pm_put() to release the reference again.
3155  */
3156 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
3157 {
3158         struct pci_dev *pdev = dev_priv->drm.pdev;
3159         struct device *kdev = &pdev->dev;
3160
3161         if (IS_ENABLED(CONFIG_PM)) {
3162                 int ret = pm_runtime_get_if_in_use(kdev);
3163
3164                 /*
3165                  * In cases runtime PM is disabled by the RPM core and we get
3166                  * an -EINVAL return value we are not supposed to call this
3167                  * function, since the power state is undefined. This applies
3168                  * atm to the late/early system suspend/resume handlers.
3169                  */
3170                 WARN_ONCE(ret < 0,
3171                           "pm_runtime_get_if_in_use() failed: %d\n", ret);
3172                 if (ret <= 0)
3173                         return false;
3174         }
3175
3176         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3177         assert_rpm_wakelock_held(dev_priv);
3178
3179         return true;
3180 }
3181
3182 /**
3183  * intel_runtime_pm_get_noresume - grab a runtime pm reference
3184  * @dev_priv: i915 device instance
3185  *
3186  * This function grabs a device-level runtime pm reference (mostly used for GEM
3187  * code to ensure the GTT or GT is on).
3188  *
3189  * It will _not_ power up the device but instead only check that it's powered
3190  * on.  Therefore it is only valid to call this functions from contexts where
3191  * the device is known to be powered up and where trying to power it up would
3192  * result in hilarity and deadlocks. That pretty much means only the system
3193  * suspend/resume code where this is used to grab runtime pm references for
3194  * delayed setup down in work items.
3195  *
3196  * Any runtime pm reference obtained by this function must have a symmetric
3197  * call to intel_runtime_pm_put() to release the reference again.
3198  */
3199 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
3200 {
3201         struct pci_dev *pdev = dev_priv->drm.pdev;
3202         struct device *kdev = &pdev->dev;
3203
3204         assert_rpm_wakelock_held(dev_priv);
3205         pm_runtime_get_noresume(kdev);
3206
3207         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3208 }
3209
3210 /**
3211  * intel_runtime_pm_put - release a runtime pm reference
3212  * @dev_priv: i915 device instance
3213  *
3214  * This function drops the device-level runtime pm reference obtained by
3215  * intel_runtime_pm_get() and might power down the corresponding
3216  * hardware block right away if this is the last reference.
3217  */
3218 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
3219 {
3220         struct pci_dev *pdev = dev_priv->drm.pdev;
3221         struct device *kdev = &pdev->dev;
3222
3223         assert_rpm_wakelock_held(dev_priv);
3224         atomic_dec(&dev_priv->runtime_pm.wakeref_count);
3225
3226         pm_runtime_mark_last_busy(kdev);
3227         pm_runtime_put_autosuspend(kdev);
3228 }
3229
3230 /**
3231  * intel_runtime_pm_enable - enable runtime pm
3232  * @dev_priv: i915 device instance
3233  *
3234  * This function enables runtime pm at the end of the driver load sequence.
3235  *
3236  * Note that this function does currently not enable runtime pm for the
3237  * subordinate display power domains. That is only done on the first modeset
3238  * using intel_display_set_init_power().
3239  */
3240 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
3241 {
3242         struct pci_dev *pdev = dev_priv->drm.pdev;
3243         struct device *kdev = &pdev->dev;
3244
3245         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
3246         pm_runtime_mark_last_busy(kdev);
3247
3248         /*
3249          * Take a permanent reference to disable the RPM functionality and drop
3250          * it only when unloading the driver. Use the low level get/put helpers,
3251          * so the driver's own RPM reference tracking asserts also work on
3252          * platforms without RPM support.
3253          */
3254         if (!HAS_RUNTIME_PM(dev_priv)) {
3255                 int ret;
3256
3257                 pm_runtime_dont_use_autosuspend(kdev);
3258                 ret = pm_runtime_get_sync(kdev);
3259                 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3260         } else {
3261                 pm_runtime_use_autosuspend(kdev);
3262         }
3263
3264         /*
3265          * The core calls the driver load handler with an RPM reference held.
3266          * We drop that here and will reacquire it during unloading in
3267          * intel_power_domains_fini().
3268          */
3269         pm_runtime_put_autosuspend(kdev);
3270 }