drivers/gpu/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *    Daniel Vetter <daniel.vetter@ffwll.ch>
  26  *
  27  */
  28
  29 #include <linux/pm_runtime.h>
  30 #include <linux/vgaarb.h>
  31
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 /**
  36  * DOC: runtime pm
  37  *
  38  * The i915 driver supports dynamic enabling and disabling of entire hardware
  39  * blocks at runtime. This is especially important on the display side where
  40  * software is supposed to control many power gates manually on recent hardware,
  41  * since on the GT side a lot of the power management is done by the hardware.
  42  * But even there some manual control at the device level is required.
  43  *
  44  * Since i915 supports a diverse set of platforms with a unified codebase and
  45  * hardware engineers just love to shuffle functionality around between power
  46  * domains there's a sizeable amount of indirection required. This file provides
  47  * generic functions to the driver for grabbing and releasing references for
  48  * abstract power domains. It then maps those to the actual power wells
  49  * present for a given platform.
  50  */
  51
  52 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  53                                          enum i915_power_well_id power_well_id);
  54
  55 static struct i915_power_well *
  56 lookup_power_well(struct drm_i915_private *dev_priv,
  57                   enum i915_power_well_id power_well_id);
  58
  59 const char *
  60 intel_display_power_domain_str(enum intel_display_power_domain domain)
  61 {
  62         switch (domain) {
  63         case POWER_DOMAIN_PIPE_A:
  64                 return "PIPE_A";
  65         case POWER_DOMAIN_PIPE_B:
  66                 return "PIPE_B";
  67         case POWER_DOMAIN_PIPE_C:
  68                 return "PIPE_C";
  69         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  70                 return "PIPE_A_PANEL_FITTER";
  71         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  72                 return "PIPE_B_PANEL_FITTER";
  73         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  74                 return "PIPE_C_PANEL_FITTER";
  75         case POWER_DOMAIN_TRANSCODER_A:
  76                 return "TRANSCODER_A";
  77         case POWER_DOMAIN_TRANSCODER_B:
  78                 return "TRANSCODER_B";
  79         case POWER_DOMAIN_TRANSCODER_C:
  80                 return "TRANSCODER_C";
  81         case POWER_DOMAIN_TRANSCODER_EDP:
  82                 return "TRANSCODER_EDP";
  83         case POWER_DOMAIN_TRANSCODER_DSI_A:
  84                 return "TRANSCODER_DSI_A";
  85         case POWER_DOMAIN_TRANSCODER_DSI_C:
  86                 return "TRANSCODER_DSI_C";
  87         case POWER_DOMAIN_PORT_DDI_A_LANES:
  88                 return "PORT_DDI_A_LANES";
  89         case POWER_DOMAIN_PORT_DDI_B_LANES:
  90                 return "PORT_DDI_B_LANES";
  91         case POWER_DOMAIN_PORT_DDI_C_LANES:
  92                 return "PORT_DDI_C_LANES";
  93         case POWER_DOMAIN_PORT_DDI_D_LANES:
  94                 return "PORT_DDI_D_LANES";
  95         case POWER_DOMAIN_PORT_DDI_E_LANES:
  96                 return "PORT_DDI_E_LANES";
  97         case POWER_DOMAIN_PORT_DDI_A_IO:
  98                 return "PORT_DDI_A_IO";
  99         case POWER_DOMAIN_PORT_DDI_B_IO:
 100                 return "PORT_DDI_B_IO";
 101         case POWER_DOMAIN_PORT_DDI_C_IO:
 102                 return "PORT_DDI_C_IO";
 103         case POWER_DOMAIN_PORT_DDI_D_IO:
 104                 return "PORT_DDI_D_IO";
 105         case POWER_DOMAIN_PORT_DDI_E_IO:
 106                 return "PORT_DDI_E_IO";
 107         case POWER_DOMAIN_PORT_DSI:
 108                 return "PORT_DSI";
 109         case POWER_DOMAIN_PORT_CRT:
 110                 return "PORT_CRT";
 111         case POWER_DOMAIN_PORT_OTHER:
 112                 return "PORT_OTHER";
 113         case POWER_DOMAIN_VGA:
 114                 return "VGA";
 115         case POWER_DOMAIN_AUDIO:
 116                 return "AUDIO";
 117         case POWER_DOMAIN_PLLS:
 118                 return "PLLS";
 119         case POWER_DOMAIN_AUX_A:
 120                 return "AUX_A";
 121         case POWER_DOMAIN_AUX_B:
 122                 return "AUX_B";
 123         case POWER_DOMAIN_AUX_C:
 124                 return "AUX_C";
 125         case POWER_DOMAIN_AUX_D:
 126                 return "AUX_D";
 127         case POWER_DOMAIN_GMBUS:
 128                 return "GMBUS";
 129         case POWER_DOMAIN_INIT:
 130                 return "INIT";
 131         case POWER_DOMAIN_MODESET:
 132                 return "MODESET";
 133         case POWER_DOMAIN_GT_IRQ:
 134                 return "GT_IRQ";
 135         default:
 136                 MISSING_CASE(domain);
 137                 return "?";
 138         }
 139 }
 140
 141 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 142                                     struct i915_power_well *power_well)
 143 {
 144         DRM_DEBUG_KMS("enabling %s\n", power_well->name);
 145         power_well->ops->enable(dev_priv, power_well);
 146         power_well->hw_enabled = true;
 147 }
 148
 149 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 150                                      struct i915_power_well *power_well)
 151 {
 152         DRM_DEBUG_KMS("disabling %s\n", power_well->name);
 153         power_well->hw_enabled = false;
 154         power_well->ops->disable(dev_priv, power_well);
 155 }
 156
 157 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 158                                  struct i915_power_well *power_well)
 159 {
 160         if (!power_well->count++)
 161                 intel_power_well_enable(dev_priv, power_well);
 162 }
 163
 164 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 165                                  struct i915_power_well *power_well)
 166 {
 167         WARN(!power_well->count, "Use count on power well %s is already zero",
 168              power_well->name);
 169
 170         if (!--power_well->count)
 171                 intel_power_well_disable(dev_priv, power_well);
 172 }
 173
 174 /**
 175  * __intel_display_power_is_enabled - unlocked check for a power domain
 176  * @dev_priv: i915 device instance
 177  * @domain: power domain to check
 178  *
 179  * This is the unlocked version of intel_display_power_is_enabled() and should
 180  * only be used from error capture and recovery code where deadlocks are
 181  * possible.
 182  *
 183  * Returns:
 184  * True when the power domain is enabled, false otherwise.
 185  */
 186 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 187                                       enum intel_display_power_domain domain)
 188 {
 189         struct i915_power_well *power_well;
 190         bool is_enabled;
 191
 192         if (dev_priv->runtime_pm.suspended)
 193                 return false;
 194
 195         is_enabled = true;
 196
 197         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
 198                 if (power_well->always_on)
 199                         continue;
 200
 201                 if (!power_well->hw_enabled) {
 202                         is_enabled = false;
 203                         break;
 204                 }
 205         }
 206
 207         return is_enabled;
 208 }
 209
 210 /**
 211  * intel_display_power_is_enabled - check for a power domain
 212  * @dev_priv: i915 device instance
 213  * @domain: power domain to check
 214  *
 215  * This function can be used to check the hw power domain state. It is mostly
 216  * used in hardware state readout functions. Everywhere else code should rely
 217  * upon explicit power domain reference counting to ensure that the hardware
 218  * block is powered up before accessing it.
 219  *
 220  * Callers must hold the relevant modesetting locks to ensure that concurrent
 221  * threads can't disable the power well while the caller tries to read a few
 222  * registers.
 223  *
 224  * Returns:
 225  * True when the power domain is enabled, false otherwise.
 226  */
 227 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 228                                     enum intel_display_power_domain domain)
 229 {
 230         struct i915_power_domains *power_domains;
 231         bool ret;
 232
 233         power_domains = &dev_priv->power_domains;
 234
 235         mutex_lock(&power_domains->lock);
 236         ret = __intel_display_power_is_enabled(dev_priv, domain);
 237         mutex_unlock(&power_domains->lock);
 238
 239         return ret;
 240 }
 241
 242 /**
 243  * intel_display_set_init_power - set the initial power domain state
 244  * @dev_priv: i915 device instance
 245  * @enable: whether to enable or disable the initial power domain state
 246  *
 247  * For simplicity our driver load/unload and system suspend/resume code assumes
 248  * that all power domains are always enabled. This functions controls the state
 249  * of this little hack. While the initial power domain state is enabled runtime
 250  * pm is effectively disabled.
 251  */
 252 void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 253                                   bool enable)
 254 {
 255         if (dev_priv->power_domains.init_power_on == enable)
 256                 return;
 257
 258         if (enable)
 259                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 260         else
 261                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 262
 263         dev_priv->power_domains.init_power_on = enable;
 264 }
 265
 266 /*
 267  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 268  * when not needed anymore. We have 4 registers that can request the power well
 269  * to be enabled, and it will only be disabled if none of the registers is
 270  * requesting it to be enabled.
 271  */
 272 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
 273                                        u8 irq_pipe_mask, bool has_vga)
 274 {
 275         struct pci_dev *pdev = dev_priv->drm.pdev;
 276
 277         /*
 278          * After we re-enable the power well, if we touch VGA register 0x3d5
 279          * we'll get unclaimed register interrupts. This stops after we write
 280          * anything to the VGA MSR register. The vgacon module uses this
 281          * register all the time, so if we unbind our driver and, as a
 282          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 283          * console_unlock(). So make here we touch the VGA MSR register, making
 284          * sure vgacon can keep working normally without triggering interrupts
 285          * and error messages.
 286          */
 287         if (has_vga) {
 288                 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 289                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 290                 vga_put(pdev, VGA_RSRC_LEGACY_IO);
 291         }
 292
 293         if (irq_pipe_mask)
 294                 gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 295 }
 296
 297 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
 298                                        u8 irq_pipe_mask)
 299 {
 300         if (irq_pipe_mask)
 301                 gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 302 }
 303
 304
 305 static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 306                                            struct i915_power_well *power_well)
 307 {
 308         enum i915_power_well_id id = power_well->id;
 309
 310         /* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
 311         WARN_ON(intel_wait_for_register(dev_priv,
 312                                         HSW_PWR_WELL_CTL_DRIVER(id),
 313                                         HSW_PWR_WELL_CTL_STATE(id),
 314                                         HSW_PWR_WELL_CTL_STATE(id),
 315                                         1));
 316 }
 317
 318 static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
 319                                      enum i915_power_well_id id)
 320 {
 321         u32 req_mask = HSW_PWR_WELL_CTL_REQ(id);
 322         u32 ret;
 323
 324         ret = I915_READ(HSW_PWR_WELL_CTL_BIOS(id)) & req_mask ? 1 : 0;
 325         ret |= I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & req_mask ? 2 : 0;
 326         ret |= I915_READ(HSW_PWR_WELL_CTL_KVMR) & req_mask ? 4 : 0;
 327         ret |= I915_READ(HSW_PWR_WELL_CTL_DEBUG(id)) & req_mask ? 8 : 0;
 328
 329         return ret;
 330 }
 331
 332 static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
 333                                             struct i915_power_well *power_well)
 334 {
 335         enum i915_power_well_id id = power_well->id;
 336         bool disabled;
 337         u32 reqs;
 338
 339         /*
 340          * Bspec doesn't require waiting for PWs to get disabled, but still do
 341          * this for paranoia. The known cases where a PW will be forced on:
 342          * - a KVMR request on any power well via the KVMR request register
 343          * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
 344          *   DEBUG request registers
 345          * Skip the wait in case any of the request bits are set and print a
 346          * diagnostic message.
 347          */
 348         wait_for((disabled = !(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 349                                HSW_PWR_WELL_CTL_STATE(id))) ||
 350                  (reqs = hsw_power_well_requesters(dev_priv, id)), 1);
 351         if (disabled)
 352                 return;
 353
 354         DRM_DEBUG_KMS("%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
 355                       power_well->name,
 356                       !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 357 }
 358
 359 static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 360                                            enum skl_power_gate pg)
 361 {
 362         /* Timeout 5us for PG#0, for other PGs 1us */
 363         WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
 364                                         SKL_FUSE_PG_DIST_STATUS(pg),
 365                                         SKL_FUSE_PG_DIST_STATUS(pg), 1));
 366 }
 367
 368 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 369                                   struct i915_power_well *power_well)
 370 {
 371         enum i915_power_well_id id = power_well->id;
 372         bool wait_fuses = power_well->hsw.has_fuses;
 373         enum skl_power_gate uninitialized_var(pg);
 374         u32 val;
 375
 376         if (wait_fuses) {
 377                 pg = SKL_PW_TO_PG(id);
 378                 /*
 379                  * For PW1 we have to wait both for the PW0/PG0 fuse state
 380                  * before enabling the power well and PW1/PG1's own fuse
 381                  * state after the enabling. For all other power wells with
 382                  * fuses we only have to wait for that PW/PG's fuse state
 383                  * after the enabling.
 384                  */
 385                 if (pg == SKL_PG1)
 386                         gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 387         }
 388
 389         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 390         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
 391         hsw_wait_for_power_well_enable(dev_priv, power_well);
 392
 393         if (wait_fuses)
 394                 gen9_wait_for_power_well_fuses(dev_priv, pg);
 395
 396         hsw_power_well_post_enable(dev_priv, power_well->hsw.irq_pipe_mask,
 397                                    power_well->hsw.has_vga);
 398 }
 399
 400 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 401                                    struct i915_power_well *power_well)
 402 {
 403         enum i915_power_well_id id = power_well->id;
 404         u32 val;
 405
 406         hsw_power_well_pre_disable(dev_priv, power_well->hsw.irq_pipe_mask);
 407
 408         val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 409         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
 410                    val & ~HSW_PWR_WELL_CTL_REQ(id));
 411         hsw_wait_for_power_well_disable(dev_priv, power_well);
 412 }
 413
 414 /*
 415  * We should only use the power well if we explicitly asked the hardware to
 416  * enable it, so check if it's enabled and also check if we've requested it to
 417  * be enabled.
 418  */
 419 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 420                                    struct i915_power_well *power_well)
 421 {
 422         enum i915_power_well_id id = power_well->id;
 423         u32 mask = HSW_PWR_WELL_CTL_REQ(id) | HSW_PWR_WELL_CTL_STATE(id);
 424
 425         return (I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & mask) == mask;
 426 }
 427
 428 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 429 {
 430         enum i915_power_well_id id = SKL_DISP_PW_2;
 431
 432         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 433                   "DC9 already programmed to be enabled.\n");
 434         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 435                   "DC5 still not disabled to enable DC9.\n");
 436         WARN_ONCE(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
 437                   HSW_PWR_WELL_CTL_REQ(id),
 438                   "Power well 2 on.\n");
 439         WARN_ONCE(intel_irqs_enabled(dev_priv),
 440                   "Interrupts not disabled yet.\n");
 441
 442          /*
 443           * TODO: check for the following to verify the conditions to enter DC9
 444           * state are satisfied:
 445           * 1] Check relevant display engine registers to verify if mode set
 446           * disable sequence was followed.
 447           * 2] Check if display uninitialize sequence is initialized.
 448           */
 449 }
 450
 451 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 452 {
 453         WARN_ONCE(intel_irqs_enabled(dev_priv),
 454                   "Interrupts not disabled yet.\n");
 455         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 456                   "DC5 still not disabled.\n");
 457
 458          /*
 459           * TODO: check for the following to verify DC9 state was indeed
 460           * entered before programming to disable it:
 461           * 1] Check relevant display engine registers to verify if mode
 462           *  set disable sequence was followed.
 463           * 2] Check if display uninitialize sequence is initialized.
 464           */
 465 }
 466
 467 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 468                                 u32 state)
 469 {
 470         int rewrites = 0;
 471         int rereads = 0;
 472         u32 v;
 473
 474         I915_WRITE(DC_STATE_EN, state);
 475
 476         /* It has been observed that disabling the dc6 state sometimes
 477          * doesn't stick and dmc keeps returning old value. Make sure
 478          * the write really sticks enough times and also force rewrite until
 479          * we are confident that state is exactly what we want.
 480          */
 481         do  {
 482                 v = I915_READ(DC_STATE_EN);
 483
 484                 if (v != state) {
 485                         I915_WRITE(DC_STATE_EN, state);
 486                         rewrites++;
 487                         rereads = 0;
 488                 } else if (rereads++ > 5) {
 489                         break;
 490                 }
 491
 492         } while (rewrites < 100);
 493
 494         if (v != state)
 495                 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
 496                           state, v);
 497
 498         /* Most of the times we need one retry, avoid spam */
 499         if (rewrites > 1)
 500                 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
 501                               state, rewrites);
 502 }
 503
 504 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 505 {
 506         u32 mask;
 507
 508         mask = DC_STATE_EN_UPTO_DC5;
 509         if (IS_GEN9_LP(dev_priv))
 510                 mask |= DC_STATE_EN_DC9;
 511         else
 512                 mask |= DC_STATE_EN_UPTO_DC6;
 513
 514         return mask;
 515 }
 516
 517 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 518 {
 519         u32 val;
 520
 521         val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
 522
 523         DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
 524                       dev_priv->csr.dc_state, val);
 525         dev_priv->csr.dc_state = val;
 526 }
 527
 528 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 529 {
 530         uint32_t val;
 531         uint32_t mask;
 532
 533         if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
 534                 state &= dev_priv->csr.allowed_dc_mask;
 535
 536         val = I915_READ(DC_STATE_EN);
 537         mask = gen9_dc_mask(dev_priv);
 538         DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
 539                       val & mask, state);
 540
 541         /* Check if DMC is ignoring our DC state requests */
 542         if ((val & mask) != dev_priv->csr.dc_state)
 543                 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
 544                           dev_priv->csr.dc_state, val & mask);
 545
 546         val &= ~mask;
 547         val |= state;
 548
 549         gen9_write_dc_state(dev_priv, val);
 550
 551         dev_priv->csr.dc_state = val & mask;
 552 }
 553
 554 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 555 {
 556         assert_can_enable_dc9(dev_priv);
 557
 558         DRM_DEBUG_KMS("Enabling DC9\n");
 559
 560         intel_power_sequencer_reset(dev_priv);
 561         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 562 }
 563
 564 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 565 {
 566         assert_can_disable_dc9(dev_priv);
 567
 568         DRM_DEBUG_KMS("Disabling DC9\n");
 569
 570         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 571
 572         intel_pps_unlock_regs_wa(dev_priv);
 573 }
 574
 575 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 576 {
 577         WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
 578                   "CSR program storage start is NULL\n");
 579         WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
 580         WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 581 }
 582
 583 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 584 {
 585         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 586                                         SKL_DISP_PW_2);
 587
 588         WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 589
 590         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 591                   "DC5 already programmed to be enabled.\n");
 592         assert_rpm_wakelock_held(dev_priv);
 593
 594         assert_csr_loaded(dev_priv);
 595 }
 596
 597 void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 598 {
 599         assert_can_enable_dc5(dev_priv);
 600
 601         DRM_DEBUG_KMS("Enabling DC5\n");
 602
 603         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 604 }
 605
 606 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 607 {
 608         WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 609                   "Backlight is not disabled.\n");
 610         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 611                   "DC6 already programmed to be enabled.\n");
 612
 613         assert_csr_loaded(dev_priv);
 614 }
 615
 616 void skl_enable_dc6(struct drm_i915_private *dev_priv)
 617 {
 618         assert_can_enable_dc6(dev_priv);
 619
 620         DRM_DEBUG_KMS("Enabling DC6\n");
 621
 622         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 623
 624 }
 625
 626 void skl_disable_dc6(struct drm_i915_private *dev_priv)
 627 {
 628         DRM_DEBUG_KMS("Disabling DC6\n");
 629
 630         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 631 }
 632
 633 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 634                                    struct i915_power_well *power_well)
 635 {
 636         enum i915_power_well_id id = power_well->id;
 637         u32 mask = HSW_PWR_WELL_CTL_REQ(id);
 638         u32 bios_req = I915_READ(HSW_PWR_WELL_CTL_BIOS(id));
 639
 640         /* Take over the request bit if set by BIOS. */
 641         if (bios_req & mask) {
 642                 u32 drv_req = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 643
 644                 if (!(drv_req & mask))
 645                         I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), drv_req | mask);
 646                 I915_WRITE(HSW_PWR_WELL_CTL_BIOS(id), bios_req & ~mask);
 647         }
 648 }
 649
 650 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 651                                            struct i915_power_well *power_well)
 652 {
 653         bxt_ddi_phy_init(dev_priv, power_well->bxt.phy);
 654 }
 655
 656 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 657                                             struct i915_power_well *power_well)
 658 {
 659         bxt_ddi_phy_uninit(dev_priv, power_well->bxt.phy);
 660 }
 661
 662 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 663                                             struct i915_power_well *power_well)
 664 {
 665         return bxt_ddi_phy_is_enabled(dev_priv, power_well->bxt.phy);
 666 }
 667
 668 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 669 {
 670         struct i915_power_well *power_well;
 671
 672         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 673         if (power_well->count > 0)
 674                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 675
 676         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 677         if (power_well->count > 0)
 678                 bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 679
 680         if (IS_GEMINILAKE(dev_priv)) {
 681                 power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
 682                 if (power_well->count > 0)
 683                         bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 684         }
 685 }
 686
 687 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 688                                            struct i915_power_well *power_well)
 689 {
 690         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 691 }
 692
 693 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 694 {
 695         u32 tmp = I915_READ(DBUF_CTL);
 696
 697         WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
 698              (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
 699              "Unexpected DBuf power power state (0x%08x)\n", tmp);
 700 }
 701
 702 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 703                                           struct i915_power_well *power_well)
 704 {
 705         struct intel_cdclk_state cdclk_state = {};
 706
 707         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 708
 709         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
 710         /* Can't read out voltage_level so can't use intel_cdclk_changed() */
 711         WARN_ON(intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, &cdclk_state));
 712
 713         gen9_assert_dbuf_enabled(dev_priv);
 714
 715         if (IS_GEN9_LP(dev_priv))
 716                 bxt_verify_ddi_phy_power_wells(dev_priv);
 717 }
 718
 719 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 720                                            struct i915_power_well *power_well)
 721 {
 722         if (!dev_priv->csr.dmc_payload)
 723                 return;
 724
 725         if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 726                 skl_enable_dc6(dev_priv);
 727         else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 728                 gen9_enable_dc5(dev_priv);
 729 }
 730
 731 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
 732                                          struct i915_power_well *power_well)
 733 {
 734 }
 735
 736 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 737                                            struct i915_power_well *power_well)
 738 {
 739 }
 740
 741 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 742                                              struct i915_power_well *power_well)
 743 {
 744         return true;
 745 }
 746
 747 static void i830_pipes_power_well_enable(struct drm_i915_private *dev_priv,
 748                                          struct i915_power_well *power_well)
 749 {
 750         if ((I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE) == 0)
 751                 i830_enable_pipe(dev_priv, PIPE_A);
 752         if ((I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE) == 0)
 753                 i830_enable_pipe(dev_priv, PIPE_B);
 754 }
 755
 756 static void i830_pipes_power_well_disable(struct drm_i915_private *dev_priv,
 757                                           struct i915_power_well *power_well)
 758 {
 759         i830_disable_pipe(dev_priv, PIPE_B);
 760         i830_disable_pipe(dev_priv, PIPE_A);
 761 }
 762
 763 static bool i830_pipes_power_well_enabled(struct drm_i915_private *dev_priv,
 764                                           struct i915_power_well *power_well)
 765 {
 766         return I915_READ(PIPECONF(PIPE_A)) & PIPECONF_ENABLE &&
 767                 I915_READ(PIPECONF(PIPE_B)) & PIPECONF_ENABLE;
 768 }
 769
 770 static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
 771                                           struct i915_power_well *power_well)
 772 {
 773         if (power_well->count > 0)
 774                 i830_pipes_power_well_enable(dev_priv, power_well);
 775         else
 776                 i830_pipes_power_well_disable(dev_priv, power_well);
 777 }
 778
 779 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 780                                struct i915_power_well *power_well, bool enable)
 781 {
 782         enum i915_power_well_id power_well_id = power_well->id;
 783         u32 mask;
 784         u32 state;
 785         u32 ctrl;
 786
 787         mask = PUNIT_PWRGT_MASK(power_well_id);
 788         state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 789                          PUNIT_PWRGT_PWR_GATE(power_well_id);
 790
 791         mutex_lock(&dev_priv->pcu_lock);
 792
 793 #define COND \
 794         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 795
 796         if (COND)
 797                 goto out;
 798
 799         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 800         ctrl &= ~mask;
 801         ctrl |= state;
 802         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 803
 804         if (wait_for(COND, 100))
 805                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 806                           state,
 807                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 808
 809 #undef COND
 810
 811 out:
 812         mutex_unlock(&dev_priv->pcu_lock);
 813 }
 814
 815 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 816                                   struct i915_power_well *power_well)
 817 {
 818         vlv_set_power_well(dev_priv, power_well, true);
 819 }
 820
 821 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 822                                    struct i915_power_well *power_well)
 823 {
 824         vlv_set_power_well(dev_priv, power_well, false);
 825 }
 826
 827 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 828                                    struct i915_power_well *power_well)
 829 {
 830         enum i915_power_well_id power_well_id = power_well->id;
 831         bool enabled = false;
 832         u32 mask;
 833         u32 state;
 834         u32 ctrl;
 835
 836         mask = PUNIT_PWRGT_MASK(power_well_id);
 837         ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
 838
 839         mutex_lock(&dev_priv->pcu_lock);
 840
 841         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
 842         /*
 843          * We only ever set the power-on and power-gate states, anything
 844          * else is unexpected.
 845          */
 846         WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
 847                 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
 848         if (state == ctrl)
 849                 enabled = true;
 850
 851         /*
 852          * A transient state at this point would mean some unexpected party
 853          * is poking at the power controls too.
 854          */
 855         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 856         WARN_ON(ctrl != state);
 857
 858         mutex_unlock(&dev_priv->pcu_lock);
 859
 860         return enabled;
 861 }
 862
 863 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 864 {
 865         u32 val;
 866
 867         /*
 868          * On driver load, a pipe may be active and driving a DSI display.
 869          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
 870          * (and never recovering) in this case. intel_dsi_post_disable() will
 871          * clear it when we turn off the display.
 872          */
 873         val = I915_READ(DSPCLK_GATE_D);
 874         val &= DPOUNIT_CLOCK_GATE_DISABLE;
 875         val |= VRHUNIT_CLOCK_GATE_DISABLE;
 876         I915_WRITE(DSPCLK_GATE_D, val);
 877
 878         /*
 879          * Disable trickle feed and enable pnd deadline calculation
 880          */
 881         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 882         I915_WRITE(CBR1_VLV, 0);
 883
 884         WARN_ON(dev_priv->rawclk_freq == 0);
 885
 886         I915_WRITE(RAWCLK_FREQ_VLV,
 887                    DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
 888 }
 889
 890 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 891 {
 892         struct intel_encoder *encoder;
 893         enum pipe pipe;
 894
 895         /*
 896          * Enable the CRI clock source so we can get at the
 897          * display and the reference clock for VGA
 898          * hotplug / manual detection. Supposedly DSI also
 899          * needs the ref clock up and running.
 900          *
 901          * CHV DPLL B/C have some issues if VGA mode is enabled.
 902          */
 903         for_each_pipe(dev_priv, pipe) {
 904                 u32 val = I915_READ(DPLL(pipe));
 905
 906                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
 907                 if (pipe != PIPE_A)
 908                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
 909
 910                 I915_WRITE(DPLL(pipe), val);
 911         }
 912
 913         vlv_init_display_clock_gating(dev_priv);
 914
 915         spin_lock_irq(&dev_priv->irq_lock);
 916         valleyview_enable_display_irqs(dev_priv);
 917         spin_unlock_irq(&dev_priv->irq_lock);
 918
 919         /*
 920          * During driver initialization/resume we can avoid restoring the
 921          * part of the HW/SW state that will be inited anyway explicitly.
 922          */
 923         if (dev_priv->power_domains.initializing)
 924                 return;
 925
 926         intel_hpd_init(dev_priv);
 927
 928         /* Re-enable the ADPA, if we have one */
 929         for_each_intel_encoder(&dev_priv->drm, encoder) {
 930                 if (encoder->type == INTEL_OUTPUT_ANALOG)
 931                         intel_crt_reset(&encoder->base);
 932         }
 933
 934         i915_redisable_vga_power_on(dev_priv);
 935
 936         intel_pps_unlock_regs_wa(dev_priv);
 937 }
 938
 939 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
 940 {
 941         spin_lock_irq(&dev_priv->irq_lock);
 942         valleyview_disable_display_irqs(dev_priv);
 943         spin_unlock_irq(&dev_priv->irq_lock);
 944
 945         /* make sure we're done processing display irqs */
 946         synchronize_irq(dev_priv->drm.irq);
 947
 948         intel_power_sequencer_reset(dev_priv);
 949
 950         /* Prevent us from re-enabling polling on accident in late suspend */
 951         if (!dev_priv->drm.dev->power.is_suspended)
 952                 intel_hpd_poll_init(dev_priv);
 953 }
 954
 955 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 956                                           struct i915_power_well *power_well)
 957 {
 958         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 959
 960         vlv_set_power_well(dev_priv, power_well, true);
 961
 962         vlv_display_power_well_init(dev_priv);
 963 }
 964
 965 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 966                                            struct i915_power_well *power_well)
 967 {
 968         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
 969
 970         vlv_display_power_well_deinit(dev_priv);
 971
 972         vlv_set_power_well(dev_priv, power_well, false);
 973 }
 974
 975 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 976                                            struct i915_power_well *power_well)
 977 {
 978         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
 979
 980         /* since ref/cri clock was enabled */
 981         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 982
 983         vlv_set_power_well(dev_priv, power_well, true);
 984
 985         /*
 986          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
 987          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
 988          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
 989          *   b. The other bits such as sfr settings / modesel may all
 990          *      be set to 0.
 991          *
 992          * This should only be done on init and resume from S3 with
 993          * both PLLs disabled, or we risk losing DPIO and PLL
 994          * synchronization.
 995          */
 996         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
 997 }
 998
 999 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1000                                             struct i915_power_well *power_well)
1001 {
1002         enum pipe pipe;
1003
1004         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1005
1006         for_each_pipe(dev_priv, pipe)
1007                 assert_pll_disabled(dev_priv, pipe);
1008
1009         /* Assert common reset */
1010         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1011
1012         vlv_set_power_well(dev_priv, power_well, false);
1013 }
1014
1015 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1016
1017 static struct i915_power_well *
1018 lookup_power_well(struct drm_i915_private *dev_priv,
1019                   enum i915_power_well_id power_well_id)
1020 {
1021         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1022         int i;
1023
1024         for (i = 0; i < power_domains->power_well_count; i++) {
1025                 struct i915_power_well *power_well;
1026
1027                 power_well = &power_domains->power_wells[i];
1028                 if (power_well->id == power_well_id)
1029                         return power_well;
1030         }
1031
1032         return NULL;
1033 }
1034
1035 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1036
1037 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1038 {
1039         struct i915_power_well *cmn_bc =
1040                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1041         struct i915_power_well *cmn_d =
1042                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1043         u32 phy_control = dev_priv->chv_phy_control;
1044         u32 phy_status = 0;
1045         u32 phy_status_mask = 0xffffffff;
1046
1047         /*
1048          * The BIOS can leave the PHY is some weird state
1049          * where it doesn't fully power down some parts.
1050          * Disable the asserts until the PHY has been fully
1051          * reset (ie. the power well has been disabled at
1052          * least once).
1053          */
1054         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1055                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1056                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1057                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1058                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1059                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1060                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1061
1062         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1063                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1064                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1065                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1066
1067         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
1068                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1069
1070                 /* this assumes override is only used to enable lanes */
1071                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1072                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1073
1074                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1075                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1076
1077                 /* CL1 is on whenever anything is on in either channel */
1078                 if (BITS_SET(phy_control,
1079                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1080                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1081                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1082
1083                 /*
1084                  * The DPLLB check accounts for the pipe B + port A usage
1085                  * with CL2 powered up but all the lanes in the second channel
1086                  * powered down.
1087                  */
1088                 if (BITS_SET(phy_control,
1089                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1090                     (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1091                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1092
1093                 if (BITS_SET(phy_control,
1094                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1095                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1096                 if (BITS_SET(phy_control,
1097                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1098                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1099
1100                 if (BITS_SET(phy_control,
1101                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1102                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1103                 if (BITS_SET(phy_control,
1104                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1105                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1106         }
1107
1108         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
1109                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1110
1111                 /* this assumes override is only used to enable lanes */
1112                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1113                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1114
1115                 if (BITS_SET(phy_control,
1116                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1117                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1118
1119                 if (BITS_SET(phy_control,
1120                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1121                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1122                 if (BITS_SET(phy_control,
1123                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1124                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1125         }
1126
1127         phy_status &= phy_status_mask;
1128
1129         /*
1130          * The PHY may be busy with some initial calibration and whatnot,
1131          * so the power state can take a while to actually change.
1132          */
1133         if (intel_wait_for_register(dev_priv,
1134                                     DISPLAY_PHY_STATUS,
1135                                     phy_status_mask,
1136                                     phy_status,
1137                                     10))
1138                 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1139                           I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1140                            phy_status, dev_priv->chv_phy_control);
1141 }
1142
1143 #undef BITS_SET
1144
1145 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1146                                            struct i915_power_well *power_well)
1147 {
1148         enum dpio_phy phy;
1149         enum pipe pipe;
1150         uint32_t tmp;
1151
1152         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1153                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1154
1155         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1156                 pipe = PIPE_A;
1157                 phy = DPIO_PHY0;
1158         } else {
1159                 pipe = PIPE_C;
1160                 phy = DPIO_PHY1;
1161         }
1162
1163         /* since ref/cri clock was enabled */
1164         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1165         vlv_set_power_well(dev_priv, power_well, true);
1166
1167         /* Poll for phypwrgood signal */
1168         if (intel_wait_for_register(dev_priv,
1169                                     DISPLAY_PHY_STATUS,
1170                                     PHY_POWERGOOD(phy),
1171                                     PHY_POWERGOOD(phy),
1172                                     1))
1173                 DRM_ERROR("Display PHY %d is not power up\n", phy);
1174
1175         mutex_lock(&dev_priv->sb_lock);
1176
1177         /* Enable dynamic power down */
1178         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1179         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1180                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1181         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1182
1183         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1184                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1185                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1186                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1187         } else {
1188                 /*
1189                  * Force the non-existing CL2 off. BXT does this
1190                  * too, so maybe it saves some power even though
1191                  * CL2 doesn't exist?
1192                  */
1193                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1194                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1195                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1196         }
1197
1198         mutex_unlock(&dev_priv->sb_lock);
1199
1200         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1201         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1202
1203         DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1204                       phy, dev_priv->chv_phy_control);
1205
1206         assert_chv_phy_status(dev_priv);
1207 }
1208
1209 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1210                                             struct i915_power_well *power_well)
1211 {
1212         enum dpio_phy phy;
1213
1214         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1215                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1216
1217         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1218                 phy = DPIO_PHY0;
1219                 assert_pll_disabled(dev_priv, PIPE_A);
1220                 assert_pll_disabled(dev_priv, PIPE_B);
1221         } else {
1222                 phy = DPIO_PHY1;
1223                 assert_pll_disabled(dev_priv, PIPE_C);
1224         }
1225
1226         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1227         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1228
1229         vlv_set_power_well(dev_priv, power_well, false);
1230
1231         DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1232                       phy, dev_priv->chv_phy_control);
1233
1234         /* PHY is fully reset now, so we can enable the PHY state asserts */
1235         dev_priv->chv_phy_assert[phy] = true;
1236
1237         assert_chv_phy_status(dev_priv);
1238 }
1239
1240 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1241                                      enum dpio_channel ch, bool override, unsigned int mask)
1242 {
1243         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1244         u32 reg, val, expected, actual;
1245
1246         /*
1247          * The BIOS can leave the PHY is some weird state
1248          * where it doesn't fully power down some parts.
1249          * Disable the asserts until the PHY has been fully
1250          * reset (ie. the power well has been disabled at
1251          * least once).
1252          */
1253         if (!dev_priv->chv_phy_assert[phy])
1254                 return;
1255
1256         if (ch == DPIO_CH0)
1257                 reg = _CHV_CMN_DW0_CH0;
1258         else
1259                 reg = _CHV_CMN_DW6_CH1;
1260
1261         mutex_lock(&dev_priv->sb_lock);
1262         val = vlv_dpio_read(dev_priv, pipe, reg);
1263         mutex_unlock(&dev_priv->sb_lock);
1264
1265         /*
1266          * This assumes !override is only used when the port is disabled.
1267          * All lanes should power down even without the override when
1268          * the port is disabled.
1269          */
1270         if (!override || mask == 0xf) {
1271                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1272                 /*
1273                  * If CH1 common lane is not active anymore
1274                  * (eg. for pipe B DPLL) the entire channel will
1275                  * shut down, which causes the common lane registers
1276                  * to read as 0. That means we can't actually check
1277                  * the lane power down status bits, but as the entire
1278                  * register reads as 0 it's a good indication that the
1279                  * channel is indeed entirely powered down.
1280                  */
1281                 if (ch == DPIO_CH1 && val == 0)
1282                         expected = 0;
1283         } else if (mask != 0x0) {
1284                 expected = DPIO_ANYDL_POWERDOWN;
1285         } else {
1286                 expected = 0;
1287         }
1288
1289         if (ch == DPIO_CH0)
1290                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1291         else
1292                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1293         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1294
1295         WARN(actual != expected,
1296              "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1297              !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1298              !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1299              reg, val);
1300 }
1301
1302 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1303                           enum dpio_channel ch, bool override)
1304 {
1305         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1306         bool was_override;
1307
1308         mutex_lock(&power_domains->lock);
1309
1310         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1311
1312         if (override == was_override)
1313                 goto out;
1314
1315         if (override)
1316                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1317         else
1318                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1319
1320         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1321
1322         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1323                       phy, ch, dev_priv->chv_phy_control);
1324
1325         assert_chv_phy_status(dev_priv);
1326
1327 out:
1328         mutex_unlock(&power_domains->lock);
1329
1330         return was_override;
1331 }
1332
1333 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1334                              bool override, unsigned int mask)
1335 {
1336         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1337         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1338         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1339         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1340
1341         mutex_lock(&power_domains->lock);
1342
1343         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1344         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1345
1346         if (override)
1347                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1348         else
1349                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1350
1351         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1352
1353         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1354                       phy, ch, mask, dev_priv->chv_phy_control);
1355
1356         assert_chv_phy_status(dev_priv);
1357
1358         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1359
1360         mutex_unlock(&power_domains->lock);
1361 }
1362
1363 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1364                                         struct i915_power_well *power_well)
1365 {
1366         enum pipe pipe = PIPE_A;
1367         bool enabled;
1368         u32 state, ctrl;
1369
1370         mutex_lock(&dev_priv->pcu_lock);
1371
1372         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1373         /*
1374          * We only ever set the power-on and power-gate states, anything
1375          * else is unexpected.
1376          */
1377         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1378         enabled = state == DP_SSS_PWR_ON(pipe);
1379
1380         /*
1381          * A transient state at this point would mean some unexpected party
1382          * is poking at the power controls too.
1383          */
1384         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1385         WARN_ON(ctrl << 16 != state);
1386
1387         mutex_unlock(&dev_priv->pcu_lock);
1388
1389         return enabled;
1390 }
1391
1392 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1393                                     struct i915_power_well *power_well,
1394                                     bool enable)
1395 {
1396         enum pipe pipe = PIPE_A;
1397         u32 state;
1398         u32 ctrl;
1399
1400         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1401
1402         mutex_lock(&dev_priv->pcu_lock);
1403
1404 #define COND \
1405         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1406
1407         if (COND)
1408                 goto out;
1409
1410         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1411         ctrl &= ~DP_SSC_MASK(pipe);
1412         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1413         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1414
1415         if (wait_for(COND, 100))
1416                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1417                           state,
1418                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1419
1420 #undef COND
1421
1422 out:
1423         mutex_unlock(&dev_priv->pcu_lock);
1424 }
1425
1426 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1427                                        struct i915_power_well *power_well)
1428 {
1429         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1430
1431         chv_set_pipe_power_well(dev_priv, power_well, true);
1432
1433         vlv_display_power_well_init(dev_priv);
1434 }
1435
1436 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1437                                         struct i915_power_well *power_well)
1438 {
1439         WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
1440
1441         vlv_display_power_well_deinit(dev_priv);
1442
1443         chv_set_pipe_power_well(dev_priv, power_well, false);
1444 }
1445
1446 static void
1447 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1448                                  enum intel_display_power_domain domain)
1449 {
1450         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1451         struct i915_power_well *power_well;
1452
1453         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
1454                 intel_power_well_get(dev_priv, power_well);
1455
1456         power_domains->domain_use_count[domain]++;
1457 }
1458
1459 /**
1460  * intel_display_power_get - grab a power domain reference
1461  * @dev_priv: i915 device instance
1462  * @domain: power domain to reference
1463  *
1464  * This function grabs a power domain reference for @domain and ensures that the
1465  * power domain and all its parents are powered up. Therefore users should only
1466  * grab a reference to the innermost power domain they need.
1467  *
1468  * Any power domain reference obtained by this function must have a symmetric
1469  * call to intel_display_power_put() to release the reference again.
1470  */
1471 void intel_display_power_get(struct drm_i915_private *dev_priv,
1472                              enum intel_display_power_domain domain)
1473 {
1474         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1475
1476         intel_runtime_pm_get(dev_priv);
1477
1478         mutex_lock(&power_domains->lock);
1479
1480         __intel_display_power_get_domain(dev_priv, domain);
1481
1482         mutex_unlock(&power_domains->lock);
1483 }
1484
1485 /**
1486  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1487  * @dev_priv: i915 device instance
1488  * @domain: power domain to reference
1489  *
1490  * This function grabs a power domain reference for @domain and ensures that the
1491  * power domain and all its parents are powered up. Therefore users should only
1492  * grab a reference to the innermost power domain they need.
1493  *
1494  * Any power domain reference obtained by this function must have a symmetric
1495  * call to intel_display_power_put() to release the reference again.
1496  */
1497 bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1498                                         enum intel_display_power_domain domain)
1499 {
1500         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1501         bool is_enabled;
1502
1503         if (!intel_runtime_pm_get_if_in_use(dev_priv))
1504                 return false;
1505
1506         mutex_lock(&power_domains->lock);
1507
1508         if (__intel_display_power_is_enabled(dev_priv, domain)) {
1509                 __intel_display_power_get_domain(dev_priv, domain);
1510                 is_enabled = true;
1511         } else {
1512                 is_enabled = false;
1513         }
1514
1515         mutex_unlock(&power_domains->lock);
1516
1517         if (!is_enabled)
1518                 intel_runtime_pm_put(dev_priv);
1519
1520         return is_enabled;
1521 }
1522
1523 /**
1524  * intel_display_power_put - release a power domain reference
1525  * @dev_priv: i915 device instance
1526  * @domain: power domain to reference
1527  *
1528  * This function drops the power domain reference obtained by
1529  * intel_display_power_get() and might power down the corresponding hardware
1530  * block right away if this is the last reference.
1531  */
1532 void intel_display_power_put(struct drm_i915_private *dev_priv,
1533                              enum intel_display_power_domain domain)
1534 {
1535         struct i915_power_domains *power_domains;
1536         struct i915_power_well *power_well;
1537
1538         power_domains = &dev_priv->power_domains;
1539
1540         mutex_lock(&power_domains->lock);
1541
1542         WARN(!power_domains->domain_use_count[domain],
1543              "Use count on domain %s is already zero\n",
1544              intel_display_power_domain_str(domain));
1545         power_domains->domain_use_count[domain]--;
1546
1547         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
1548                 intel_power_well_put(dev_priv, power_well);
1549
1550         mutex_unlock(&power_domains->lock);
1551
1552         intel_runtime_pm_put(dev_priv);
1553 }
1554
1555 #define I830_PIPES_POWER_DOMAINS (              \
1556         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1557         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1558         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1559         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1560         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1561         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1562         BIT_ULL(POWER_DOMAIN_INIT))
1563
1564 #define VLV_DISPLAY_POWER_DOMAINS (             \
1565         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1566         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1567         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1568         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1569         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1570         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1571         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1572         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1573         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1574         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1575         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1576         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1577         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1578         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1579         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1580         BIT_ULL(POWER_DOMAIN_INIT))
1581
1582 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1583         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1584         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1585         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1586         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1587         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1588         BIT_ULL(POWER_DOMAIN_INIT))
1589
1590 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1591         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1592         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1593         BIT_ULL(POWER_DOMAIN_INIT))
1594
1595 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1596         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1597         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1598         BIT_ULL(POWER_DOMAIN_INIT))
1599
1600 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1601         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1602         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1603         BIT_ULL(POWER_DOMAIN_INIT))
1604
1605 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1606         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1607         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1608         BIT_ULL(POWER_DOMAIN_INIT))
1609
1610 #define CHV_DISPLAY_POWER_DOMAINS (             \
1611         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1612         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1613         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
1614         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1615         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1616         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1617         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1618         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1619         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
1620         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1621         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1622         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1623         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1624         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1625         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1626         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1627         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1628         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1629         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1630         BIT_ULL(POWER_DOMAIN_INIT))
1631
1632 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1633         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1634         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1635         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1636         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1637         BIT_ULL(POWER_DOMAIN_INIT))
1638
1639 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1640         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1641         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1642         BIT_ULL(POWER_DOMAIN_INIT))
1643
1644 #define HSW_DISPLAY_POWER_DOMAINS (                     \
1645         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1646         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1647         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
1648         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1649         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1650         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1651         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1652         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1653         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1654         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1655         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1656         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1657         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1658         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1659         BIT_ULL(POWER_DOMAIN_INIT))
1660
1661 #define BDW_DISPLAY_POWER_DOMAINS (                     \
1662         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1663         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1664         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1665         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1666         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1667         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1668         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1669         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1670         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1671         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1672         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1673         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1674         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1675         BIT_ULL(POWER_DOMAIN_INIT))
1676
1677 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1678         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1679         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1680         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1681         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1682         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1683         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1684         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1685         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1686         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1687         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1688         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
1689         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1690         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1691         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1692         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1693         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1694         BIT_ULL(POWER_DOMAIN_INIT))
1695 #define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (          \
1696         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1697         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
1698         BIT_ULL(POWER_DOMAIN_INIT))
1699 #define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1700         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1701         BIT_ULL(POWER_DOMAIN_INIT))
1702 #define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1703         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1704         BIT_ULL(POWER_DOMAIN_INIT))
1705 #define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (            \
1706         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1707         BIT_ULL(POWER_DOMAIN_INIT))
1708 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1709         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1710         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1711         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1712         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1713         BIT_ULL(POWER_DOMAIN_INIT))
1714
1715 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1716         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1717         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1718         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1719         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1720         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1721         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1722         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1723         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1724         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1725         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1726         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1727         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1728         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1729         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
1730         BIT_ULL(POWER_DOMAIN_INIT))
1731 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1732         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1733         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1734         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1735         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1736         BIT_ULL(POWER_DOMAIN_INIT))
1737 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
1738         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1739         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1740         BIT_ULL(POWER_DOMAIN_INIT))
1741 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
1742         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1743         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1744         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1745         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1746         BIT_ULL(POWER_DOMAIN_INIT))
1747
1748 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1749         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1750         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1751         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1752         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1753         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1754         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1755         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1756         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1757         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1758         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1759         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1760         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1761         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1762         BIT_ULL(POWER_DOMAIN_INIT))
1763 #define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (            \
1764         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
1765 #define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (            \
1766         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
1767 #define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (            \
1768         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
1769 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
1770         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
1771         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1772         BIT_ULL(POWER_DOMAIN_INIT))
1773 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
1774         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1775         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1776         BIT_ULL(POWER_DOMAIN_INIT))
1777 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
1778         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1779         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1780         BIT_ULL(POWER_DOMAIN_INIT))
1781 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
1782         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
1783         BIT_ULL(POWER_DOMAIN_INIT))
1784 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
1785         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1786         BIT_ULL(POWER_DOMAIN_INIT))
1787 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
1788         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1789         BIT_ULL(POWER_DOMAIN_INIT))
1790 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1791         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1792         BIT_ULL(POWER_DOMAIN_GT_IRQ) |                  \
1793         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1794         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1795         BIT_ULL(POWER_DOMAIN_INIT))
1796
1797 #define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
1798         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1799         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1800         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1801         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1802         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1803         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1804         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1805         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1806         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1807         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1808         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
1809         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1810         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1811         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1812         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1813         BIT_ULL(POWER_DOMAIN_INIT))
1814 #define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (            \
1815         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |           \
1816         BIT_ULL(POWER_DOMAIN_INIT))
1817 #define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (            \
1818         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
1819         BIT_ULL(POWER_DOMAIN_INIT))
1820 #define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (            \
1821         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
1822         BIT_ULL(POWER_DOMAIN_INIT))
1823 #define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (            \
1824         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
1825         BIT_ULL(POWER_DOMAIN_INIT))
1826 #define CNL_DISPLAY_AUX_A_POWER_DOMAINS (               \
1827         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1828         BIT_ULL(POWER_DOMAIN_INIT))
1829 #define CNL_DISPLAY_AUX_B_POWER_DOMAINS (               \
1830         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
1831         BIT_ULL(POWER_DOMAIN_INIT))
1832 #define CNL_DISPLAY_AUX_C_POWER_DOMAINS (               \
1833         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
1834         BIT_ULL(POWER_DOMAIN_INIT))
1835 #define CNL_DISPLAY_AUX_D_POWER_DOMAINS (               \
1836         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
1837         BIT_ULL(POWER_DOMAIN_INIT))
1838 #define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
1839         CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
1840         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
1841         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
1842         BIT_ULL(POWER_DOMAIN_INIT))
1843
1844 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1845         .sync_hw = i9xx_power_well_sync_hw_noop,
1846         .enable = i9xx_always_on_power_well_noop,
1847         .disable = i9xx_always_on_power_well_noop,
1848         .is_enabled = i9xx_always_on_power_well_enabled,
1849 };
1850
1851 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
1852         .sync_hw = i9xx_power_well_sync_hw_noop,
1853         .enable = chv_pipe_power_well_enable,
1854         .disable = chv_pipe_power_well_disable,
1855         .is_enabled = chv_pipe_power_well_enabled,
1856 };
1857
1858 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1859         .sync_hw = i9xx_power_well_sync_hw_noop,
1860         .enable = chv_dpio_cmn_power_well_enable,
1861         .disable = chv_dpio_cmn_power_well_disable,
1862         .is_enabled = vlv_power_well_enabled,
1863 };
1864
1865 static struct i915_power_well i9xx_always_on_power_well[] = {
1866         {
1867                 .name = "always-on",
1868                 .always_on = 1,
1869                 .domains = POWER_DOMAIN_MASK,
1870                 .ops = &i9xx_always_on_power_well_ops,
1871                 .id = I915_DISP_PW_ALWAYS_ON,
1872         },
1873 };
1874
1875 static const struct i915_power_well_ops i830_pipes_power_well_ops = {
1876         .sync_hw = i830_pipes_power_well_sync_hw,
1877         .enable = i830_pipes_power_well_enable,
1878         .disable = i830_pipes_power_well_disable,
1879         .is_enabled = i830_pipes_power_well_enabled,
1880 };
1881
1882 static struct i915_power_well i830_power_wells[] = {
1883         {
1884                 .name = "always-on",
1885                 .always_on = 1,
1886                 .domains = POWER_DOMAIN_MASK,
1887                 .ops = &i9xx_always_on_power_well_ops,
1888                 .id = I915_DISP_PW_ALWAYS_ON,
1889         },
1890         {
1891                 .name = "pipes",
1892                 .domains = I830_PIPES_POWER_DOMAINS,
1893                 .ops = &i830_pipes_power_well_ops,
1894                 .id = I830_DISP_PW_PIPES,
1895         },
1896 };
1897
1898 static const struct i915_power_well_ops hsw_power_well_ops = {
1899         .sync_hw = hsw_power_well_sync_hw,
1900         .enable = hsw_power_well_enable,
1901         .disable = hsw_power_well_disable,
1902         .is_enabled = hsw_power_well_enabled,
1903 };
1904
1905 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1906         .sync_hw = i9xx_power_well_sync_hw_noop,
1907         .enable = gen9_dc_off_power_well_enable,
1908         .disable = gen9_dc_off_power_well_disable,
1909         .is_enabled = gen9_dc_off_power_well_enabled,
1910 };
1911
1912 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1913         .sync_hw = i9xx_power_well_sync_hw_noop,
1914         .enable = bxt_dpio_cmn_power_well_enable,
1915         .disable = bxt_dpio_cmn_power_well_disable,
1916         .is_enabled = bxt_dpio_cmn_power_well_enabled,
1917 };
1918
1919 static struct i915_power_well hsw_power_wells[] = {
1920         {
1921                 .name = "always-on",
1922                 .always_on = 1,
1923                 .domains = POWER_DOMAIN_MASK,
1924                 .ops = &i9xx_always_on_power_well_ops,
1925                 .id = I915_DISP_PW_ALWAYS_ON,
1926         },
1927         {
1928                 .name = "display",
1929                 .domains = HSW_DISPLAY_POWER_DOMAINS,
1930                 .ops = &hsw_power_well_ops,
1931                 .id = HSW_DISP_PW_GLOBAL,
1932                 {
1933                         .hsw.has_vga = true,
1934                 },
1935         },
1936 };
1937
1938 static struct i915_power_well bdw_power_wells[] = {
1939         {
1940                 .name = "always-on",
1941                 .always_on = 1,
1942                 .domains = POWER_DOMAIN_MASK,
1943                 .ops = &i9xx_always_on_power_well_ops,
1944                 .id = I915_DISP_PW_ALWAYS_ON,
1945         },
1946         {
1947                 .name = "display",
1948                 .domains = BDW_DISPLAY_POWER_DOMAINS,
1949                 .ops = &hsw_power_well_ops,
1950                 .id = HSW_DISP_PW_GLOBAL,
1951                 {
1952                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
1953                         .hsw.has_vga = true,
1954                 },
1955         },
1956 };
1957
1958 static const struct i915_power_well_ops vlv_display_power_well_ops = {
1959         .sync_hw = i9xx_power_well_sync_hw_noop,
1960         .enable = vlv_display_power_well_enable,
1961         .disable = vlv_display_power_well_disable,
1962         .is_enabled = vlv_power_well_enabled,
1963 };
1964
1965 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1966         .sync_hw = i9xx_power_well_sync_hw_noop,
1967         .enable = vlv_dpio_cmn_power_well_enable,
1968         .disable = vlv_dpio_cmn_power_well_disable,
1969         .is_enabled = vlv_power_well_enabled,
1970 };
1971
1972 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1973         .sync_hw = i9xx_power_well_sync_hw_noop,
1974         .enable = vlv_power_well_enable,
1975         .disable = vlv_power_well_disable,
1976         .is_enabled = vlv_power_well_enabled,
1977 };
1978
1979 static struct i915_power_well vlv_power_wells[] = {
1980         {
1981                 .name = "always-on",
1982                 .always_on = 1,
1983                 .domains = POWER_DOMAIN_MASK,
1984                 .ops = &i9xx_always_on_power_well_ops,
1985                 .id = I915_DISP_PW_ALWAYS_ON,
1986         },
1987         {
1988                 .name = "display",
1989                 .domains = VLV_DISPLAY_POWER_DOMAINS,
1990                 .id = PUNIT_POWER_WELL_DISP2D,
1991                 .ops = &vlv_display_power_well_ops,
1992         },
1993         {
1994                 .name = "dpio-tx-b-01",
1995                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1996                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
1997                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
1998                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
1999                 .ops = &vlv_dpio_power_well_ops,
2000                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
2001         },
2002         {
2003                 .name = "dpio-tx-b-23",
2004                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2005                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2006                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2007                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2008                 .ops = &vlv_dpio_power_well_ops,
2009                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
2010         },
2011         {
2012                 .name = "dpio-tx-c-01",
2013                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2014                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2015                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2016                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2017                 .ops = &vlv_dpio_power_well_ops,
2018                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
2019         },
2020         {
2021                 .name = "dpio-tx-c-23",
2022                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2023                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2024                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2025                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2026                 .ops = &vlv_dpio_power_well_ops,
2027                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
2028         },
2029         {
2030                 .name = "dpio-common",
2031                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2032                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2033                 .ops = &vlv_dpio_cmn_power_well_ops,
2034         },
2035 };
2036
2037 static struct i915_power_well chv_power_wells[] = {
2038         {
2039                 .name = "always-on",
2040                 .always_on = 1,
2041                 .domains = POWER_DOMAIN_MASK,
2042                 .ops = &i9xx_always_on_power_well_ops,
2043                 .id = I915_DISP_PW_ALWAYS_ON,
2044         },
2045         {
2046                 .name = "display",
2047                 /*
2048                  * Pipe A power well is the new disp2d well. Pipe B and C
2049                  * power wells don't actually exist. Pipe A power well is
2050                  * required for any pipe to work.
2051                  */
2052                 .domains = CHV_DISPLAY_POWER_DOMAINS,
2053                 .id = CHV_DISP_PW_PIPE_A,
2054                 .ops = &chv_pipe_power_well_ops,
2055         },
2056         {
2057                 .name = "dpio-common-bc",
2058                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2059                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2060                 .ops = &chv_dpio_cmn_power_well_ops,
2061         },
2062         {
2063                 .name = "dpio-common-d",
2064                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2065                 .id = PUNIT_POWER_WELL_DPIO_CMN_D,
2066                 .ops = &chv_dpio_cmn_power_well_ops,
2067         },
2068 };
2069
2070 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2071                                          enum i915_power_well_id power_well_id)
2072 {
2073         struct i915_power_well *power_well;
2074         bool ret;
2075
2076         power_well = lookup_power_well(dev_priv, power_well_id);
2077         ret = power_well->ops->is_enabled(dev_priv, power_well);
2078
2079         return ret;
2080 }
2081
2082 static struct i915_power_well skl_power_wells[] = {
2083         {
2084                 .name = "always-on",
2085                 .always_on = 1,
2086                 .domains = POWER_DOMAIN_MASK,
2087                 .ops = &i9xx_always_on_power_well_ops,
2088                 .id = I915_DISP_PW_ALWAYS_ON,
2089         },
2090         {
2091                 .name = "power well 1",
2092                 /* Handled by the DMC firmware */
2093                 .domains = 0,
2094                 .ops = &hsw_power_well_ops,
2095                 .id = SKL_DISP_PW_1,
2096                 {
2097                         .hsw.has_fuses = true,
2098                 },
2099         },
2100         {
2101                 .name = "MISC IO power well",
2102                 /* Handled by the DMC firmware */
2103                 .domains = 0,
2104                 .ops = &hsw_power_well_ops,
2105                 .id = SKL_DISP_PW_MISC_IO,
2106         },
2107         {
2108                 .name = "DC off",
2109                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2110                 .ops = &gen9_dc_off_power_well_ops,
2111                 .id = SKL_DISP_PW_DC_OFF,
2112         },
2113         {
2114                 .name = "power well 2",
2115                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2116                 .ops = &hsw_power_well_ops,
2117                 .id = SKL_DISP_PW_2,
2118                 {
2119                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2120                         .hsw.has_vga = true,
2121                         .hsw.has_fuses = true,
2122                 },
2123         },
2124         {
2125                 .name = "DDI A/E IO power well",
2126                 .domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
2127                 .ops = &hsw_power_well_ops,
2128                 .id = SKL_DISP_PW_DDI_A_E,
2129         },
2130         {
2131                 .name = "DDI B IO power well",
2132                 .domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2133                 .ops = &hsw_power_well_ops,
2134                 .id = SKL_DISP_PW_DDI_B,
2135         },
2136         {
2137                 .name = "DDI C IO power well",
2138                 .domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2139                 .ops = &hsw_power_well_ops,
2140                 .id = SKL_DISP_PW_DDI_C,
2141         },
2142         {
2143                 .name = "DDI D IO power well",
2144                 .domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
2145                 .ops = &hsw_power_well_ops,
2146                 .id = SKL_DISP_PW_DDI_D,
2147         },
2148 };
2149
2150 static struct i915_power_well bxt_power_wells[] = {
2151         {
2152                 .name = "always-on",
2153                 .always_on = 1,
2154                 .domains = POWER_DOMAIN_MASK,
2155                 .ops = &i9xx_always_on_power_well_ops,
2156                 .id = I915_DISP_PW_ALWAYS_ON,
2157         },
2158         {
2159                 .name = "power well 1",
2160                 .domains = 0,
2161                 .ops = &hsw_power_well_ops,
2162                 .id = SKL_DISP_PW_1,
2163                 {
2164                         .hsw.has_fuses = true,
2165                 },
2166         },
2167         {
2168                 .name = "DC off",
2169                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2170                 .ops = &gen9_dc_off_power_well_ops,
2171                 .id = SKL_DISP_PW_DC_OFF,
2172         },
2173         {
2174                 .name = "power well 2",
2175                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2176                 .ops = &hsw_power_well_ops,
2177                 .id = SKL_DISP_PW_2,
2178                 {
2179                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2180                         .hsw.has_vga = true,
2181                         .hsw.has_fuses = true,
2182                 },
2183         },
2184         {
2185                 .name = "dpio-common-a",
2186                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2187                 .ops = &bxt_dpio_cmn_power_well_ops,
2188                 .id = BXT_DPIO_CMN_A,
2189                 {
2190                         .bxt.phy = DPIO_PHY1,
2191                 },
2192         },
2193         {
2194                 .name = "dpio-common-bc",
2195                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2196                 .ops = &bxt_dpio_cmn_power_well_ops,
2197                 .id = BXT_DPIO_CMN_BC,
2198                 {
2199                         .bxt.phy = DPIO_PHY0,
2200                 },
2201         },
2202 };
2203
2204 static struct i915_power_well glk_power_wells[] = {
2205         {
2206                 .name = "always-on",
2207                 .always_on = 1,
2208                 .domains = POWER_DOMAIN_MASK,
2209                 .ops = &i9xx_always_on_power_well_ops,
2210                 .id = I915_DISP_PW_ALWAYS_ON,
2211         },
2212         {
2213                 .name = "power well 1",
2214                 /* Handled by the DMC firmware */
2215                 .domains = 0,
2216                 .ops = &hsw_power_well_ops,
2217                 .id = SKL_DISP_PW_1,
2218                 {
2219                         .hsw.has_fuses = true,
2220                 },
2221         },
2222         {
2223                 .name = "DC off",
2224                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2225                 .ops = &gen9_dc_off_power_well_ops,
2226                 .id = SKL_DISP_PW_DC_OFF,
2227         },
2228         {
2229                 .name = "power well 2",
2230                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2231                 .ops = &hsw_power_well_ops,
2232                 .id = SKL_DISP_PW_2,
2233                 {
2234                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2235                         .hsw.has_vga = true,
2236                         .hsw.has_fuses = true,
2237                 },
2238         },
2239         {
2240                 .name = "dpio-common-a",
2241                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2242                 .ops = &bxt_dpio_cmn_power_well_ops,
2243                 .id = BXT_DPIO_CMN_A,
2244                 {
2245                         .bxt.phy = DPIO_PHY1,
2246                 },
2247         },
2248         {
2249                 .name = "dpio-common-b",
2250                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2251                 .ops = &bxt_dpio_cmn_power_well_ops,
2252                 .id = BXT_DPIO_CMN_BC,
2253                 {
2254                         .bxt.phy = DPIO_PHY0,
2255                 },
2256         },
2257         {
2258                 .name = "dpio-common-c",
2259                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2260                 .ops = &bxt_dpio_cmn_power_well_ops,
2261                 .id = GLK_DPIO_CMN_C,
2262                 {
2263                         .bxt.phy = DPIO_PHY2,
2264                 },
2265         },
2266         {
2267                 .name = "AUX A",
2268                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2269                 .ops = &hsw_power_well_ops,
2270                 .id = GLK_DISP_PW_AUX_A,
2271         },
2272         {
2273                 .name = "AUX B",
2274                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2275                 .ops = &hsw_power_well_ops,
2276                 .id = GLK_DISP_PW_AUX_B,
2277         },
2278         {
2279                 .name = "AUX C",
2280                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2281                 .ops = &hsw_power_well_ops,
2282                 .id = GLK_DISP_PW_AUX_C,
2283         },
2284         {
2285                 .name = "DDI A IO power well",
2286                 .domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
2287                 .ops = &hsw_power_well_ops,
2288                 .id = GLK_DISP_PW_DDI_A,
2289         },
2290         {
2291                 .name = "DDI B IO power well",
2292                 .domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
2293                 .ops = &hsw_power_well_ops,
2294                 .id = SKL_DISP_PW_DDI_B,
2295         },
2296         {
2297                 .name = "DDI C IO power well",
2298                 .domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
2299                 .ops = &hsw_power_well_ops,
2300                 .id = SKL_DISP_PW_DDI_C,
2301         },
2302 };
2303
2304 static struct i915_power_well cnl_power_wells[] = {
2305         {
2306                 .name = "always-on",
2307                 .always_on = 1,
2308                 .domains = POWER_DOMAIN_MASK,
2309                 .ops = &i9xx_always_on_power_well_ops,
2310                 .id = I915_DISP_PW_ALWAYS_ON,
2311         },
2312         {
2313                 .name = "power well 1",
2314                 /* Handled by the DMC firmware */
2315                 .domains = 0,
2316                 .ops = &hsw_power_well_ops,
2317                 .id = SKL_DISP_PW_1,
2318                 {
2319                         .hsw.has_fuses = true,
2320                 },
2321         },
2322         {
2323                 .name = "AUX A",
2324                 .domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
2325                 .ops = &hsw_power_well_ops,
2326                 .id = CNL_DISP_PW_AUX_A,
2327         },
2328         {
2329                 .name = "AUX B",
2330                 .domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
2331                 .ops = &hsw_power_well_ops,
2332                 .id = CNL_DISP_PW_AUX_B,
2333         },
2334         {
2335                 .name = "AUX C",
2336                 .domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
2337                 .ops = &hsw_power_well_ops,
2338                 .id = CNL_DISP_PW_AUX_C,
2339         },
2340         {
2341                 .name = "AUX D",
2342                 .domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
2343                 .ops = &hsw_power_well_ops,
2344                 .id = CNL_DISP_PW_AUX_D,
2345         },
2346         {
2347                 .name = "DC off",
2348                 .domains = CNL_DISPLAY_DC_OFF_POWER_DOMAINS,
2349                 .ops = &gen9_dc_off_power_well_ops,
2350                 .id = SKL_DISP_PW_DC_OFF,
2351         },
2352         {
2353                 .name = "power well 2",
2354                 .domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2355                 .ops = &hsw_power_well_ops,
2356                 .id = SKL_DISP_PW_2,
2357                 {
2358                         .hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
2359                         .hsw.has_vga = true,
2360                         .hsw.has_fuses = true,
2361                 },
2362         },
2363         {
2364                 .name = "DDI A IO power well",
2365                 .domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
2366                 .ops = &hsw_power_well_ops,
2367                 .id = CNL_DISP_PW_DDI_A,
2368         },
2369         {
2370                 .name = "DDI B IO power well",
2371                 .domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
2372                 .ops = &hsw_power_well_ops,
2373                 .id = SKL_DISP_PW_DDI_B,
2374         },
2375         {
2376                 .name = "DDI C IO power well",
2377                 .domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
2378                 .ops = &hsw_power_well_ops,
2379                 .id = SKL_DISP_PW_DDI_C,
2380         },
2381         {
2382                 .name = "DDI D IO power well",
2383                 .domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
2384                 .ops = &hsw_power_well_ops,
2385                 .id = SKL_DISP_PW_DDI_D,
2386         },
2387 };
2388
2389 static int
2390 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2391                                    int disable_power_well)
2392 {
2393         if (disable_power_well >= 0)
2394                 return !!disable_power_well;
2395
2396         return 1;
2397 }
2398
2399 static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2400                                     int enable_dc)
2401 {
2402         uint32_t mask;
2403         int requested_dc;
2404         int max_dc;
2405
2406         if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
2407                 max_dc = 2;
2408                 mask = 0;
2409         } else if (IS_GEN9_LP(dev_priv)) {
2410                 max_dc = 1;
2411                 /*
2412                  * DC9 has a separate HW flow from the rest of the DC states,
2413                  * not depending on the DMC firmware. It's needed by system
2414                  * suspend/resume, so allow it unconditionally.
2415                  */
2416                 mask = DC_STATE_EN_DC9;
2417         } else {
2418                 max_dc = 0;
2419                 mask = 0;
2420         }
2421
2422         if (!i915_modparams.disable_power_well)
2423                 max_dc = 0;
2424
2425         if (enable_dc >= 0 && enable_dc <= max_dc) {
2426                 requested_dc = enable_dc;
2427         } else if (enable_dc == -1) {
2428                 requested_dc = max_dc;
2429         } else if (enable_dc > max_dc && enable_dc <= 2) {
2430                 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
2431                               enable_dc, max_dc);
2432                 requested_dc = max_dc;
2433         } else {
2434                 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
2435                 requested_dc = max_dc;
2436         }
2437
2438         if (requested_dc > 1)
2439                 mask |= DC_STATE_EN_UPTO_DC6;
2440         if (requested_dc > 0)
2441                 mask |= DC_STATE_EN_UPTO_DC5;
2442
2443         DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
2444
2445         return mask;
2446 }
2447
2448 static void assert_power_well_ids_unique(struct drm_i915_private *dev_priv)
2449 {
2450         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2451         u64 power_well_ids;
2452         int i;
2453
2454         power_well_ids = 0;
2455         for (i = 0; i < power_domains->power_well_count; i++) {
2456                 enum i915_power_well_id id = power_domains->power_wells[i].id;
2457
2458                 WARN_ON(id >= sizeof(power_well_ids) * 8);
2459                 WARN_ON(power_well_ids & BIT_ULL(id));
2460                 power_well_ids |= BIT_ULL(id);
2461         }
2462 }
2463
2464 #define set_power_wells(power_domains, __power_wells) ({                \
2465         (power_domains)->power_wells = (__power_wells);                 \
2466         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
2467 })
2468
2469 /**
2470  * intel_power_domains_init - initializes the power domain structures
2471  * @dev_priv: i915 device instance
2472  *
2473  * Initializes the power domain structures for @dev_priv depending upon the
2474  * supported platform.
2475  */
2476 int intel_power_domains_init(struct drm_i915_private *dev_priv)
2477 {
2478         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2479
2480         i915_modparams.disable_power_well =
2481                 sanitize_disable_power_well_option(dev_priv,
2482                                                    i915_modparams.disable_power_well);
2483         dev_priv->csr.allowed_dc_mask =
2484                 get_allowed_dc_mask(dev_priv, i915_modparams.enable_dc);
2485
2486         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
2487
2488         mutex_init(&power_domains->lock);
2489
2490         /*
2491          * The enabling order will be from lower to higher indexed wells,
2492          * the disabling order is reversed.
2493          */
2494         if (IS_HASWELL(dev_priv)) {
2495                 set_power_wells(power_domains, hsw_power_wells);
2496         } else if (IS_BROADWELL(dev_priv)) {
2497                 set_power_wells(power_domains, bdw_power_wells);
2498         } else if (IS_GEN9_BC(dev_priv)) {
2499                 set_power_wells(power_domains, skl_power_wells);
2500         } else if (IS_CANNONLAKE(dev_priv)) {
2501                 set_power_wells(power_domains, cnl_power_wells);
2502         } else if (IS_BROXTON(dev_priv)) {
2503                 set_power_wells(power_domains, bxt_power_wells);
2504         } else if (IS_GEMINILAKE(dev_priv)) {
2505                 set_power_wells(power_domains, glk_power_wells);
2506         } else if (IS_CHERRYVIEW(dev_priv)) {
2507                 set_power_wells(power_domains, chv_power_wells);
2508         } else if (IS_VALLEYVIEW(dev_priv)) {
2509                 set_power_wells(power_domains, vlv_power_wells);
2510         } else if (IS_I830(dev_priv)) {
2511                 set_power_wells(power_domains, i830_power_wells);
2512         } else {
2513                 set_power_wells(power_domains, i9xx_always_on_power_well);
2514         }
2515
2516         assert_power_well_ids_unique(dev_priv);
2517
2518         return 0;
2519 }
2520
2521 /**
2522  * intel_power_domains_fini - finalizes the power domain structures
2523  * @dev_priv: i915 device instance
2524  *
2525  * Finalizes the power domain structures for @dev_priv depending upon the
2526  * supported platform. This function also disables runtime pm and ensures that
2527  * the device stays powered up so that the driver can be reloaded.
2528  */
2529 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
2530 {
2531         struct device *kdev = &dev_priv->drm.pdev->dev;
2532
2533         /*
2534          * The i915.ko module is still not prepared to be loaded when
2535          * the power well is not enabled, so just enable it in case
2536          * we're going to unload/reload.
2537          * The following also reacquires the RPM reference the core passed
2538          * to the driver during loading, which is dropped in
2539          * intel_runtime_pm_enable(). We have to hand back the control of the
2540          * device to the core with this reference held.
2541          */
2542         intel_display_set_init_power(dev_priv, true);
2543
2544         /* Remove the refcount we took to keep power well support disabled. */
2545         if (!i915_modparams.disable_power_well)
2546                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2547
2548         /*
2549          * Remove the refcount we took in intel_runtime_pm_enable() in case
2550          * the platform doesn't support runtime PM.
2551          */
2552         if (!HAS_RUNTIME_PM(dev_priv))
2553                 pm_runtime_put(kdev);
2554 }
2555
2556 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
2557 {
2558         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2559         struct i915_power_well *power_well;
2560
2561         mutex_lock(&power_domains->lock);
2562         for_each_power_well(dev_priv, power_well) {
2563                 power_well->ops->sync_hw(dev_priv, power_well);
2564                 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
2565                                                                      power_well);
2566         }
2567         mutex_unlock(&power_domains->lock);
2568 }
2569
2570 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
2571 {
2572         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
2573         POSTING_READ(DBUF_CTL);
2574
2575         udelay(10);
2576
2577         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
2578                 DRM_ERROR("DBuf power enable timeout\n");
2579 }
2580
2581 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
2582 {
2583         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
2584         POSTING_READ(DBUF_CTL);
2585
2586         udelay(10);
2587
2588         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
2589                 DRM_ERROR("DBuf power disable timeout!\n");
2590 }
2591
2592 static void skl_display_core_init(struct drm_i915_private *dev_priv,
2593                                    bool resume)
2594 {
2595         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2596         struct i915_power_well *well;
2597         uint32_t val;
2598
2599         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2600
2601         /* enable PCH reset handshake */
2602         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2603         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
2604
2605         /* enable PG1 and Misc I/O */
2606         mutex_lock(&power_domains->lock);
2607
2608         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2609         intel_power_well_enable(dev_priv, well);
2610
2611         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2612         intel_power_well_enable(dev_priv, well);
2613
2614         mutex_unlock(&power_domains->lock);
2615
2616         skl_init_cdclk(dev_priv);
2617
2618         gen9_dbuf_enable(dev_priv);
2619
2620         if (resume && dev_priv->csr.dmc_payload)
2621                 intel_csr_load_program(dev_priv);
2622 }
2623
2624 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
2625 {
2626         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2627         struct i915_power_well *well;
2628
2629         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2630
2631         gen9_dbuf_disable(dev_priv);
2632
2633         skl_uninit_cdclk(dev_priv);
2634
2635         /* The spec doesn't call for removing the reset handshake flag */
2636         /* disable PG1 and Misc I/O */
2637
2638         mutex_lock(&power_domains->lock);
2639
2640         /*
2641          * BSpec says to keep the MISC IO power well enabled here, only
2642          * remove our request for power well 1.
2643          * Note that even though the driver's request is removed power well 1
2644          * may stay enabled after this due to DMC's own request on it.
2645          */
2646         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2647         intel_power_well_disable(dev_priv, well);
2648
2649         mutex_unlock(&power_domains->lock);
2650
2651         usleep_range(10, 30);           /* 10 us delay per Bspec */
2652 }
2653
2654 void bxt_display_core_init(struct drm_i915_private *dev_priv,
2655                            bool resume)
2656 {
2657         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2658         struct i915_power_well *well;
2659         uint32_t val;
2660
2661         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2662
2663         /*
2664          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
2665          * or else the reset will hang because there is no PCH to respond.
2666          * Move the handshake programming to initialization sequence.
2667          * Previously was left up to BIOS.
2668          */
2669         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2670         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
2671         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2672
2673         /* Enable PG1 */
2674         mutex_lock(&power_domains->lock);
2675
2676         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2677         intel_power_well_enable(dev_priv, well);
2678
2679         mutex_unlock(&power_domains->lock);
2680
2681         bxt_init_cdclk(dev_priv);
2682
2683         gen9_dbuf_enable(dev_priv);
2684
2685         if (resume && dev_priv->csr.dmc_payload)
2686                 intel_csr_load_program(dev_priv);
2687 }
2688
2689 void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
2690 {
2691         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2692         struct i915_power_well *well;
2693
2694         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2695
2696         gen9_dbuf_disable(dev_priv);
2697
2698         bxt_uninit_cdclk(dev_priv);
2699
2700         /* The spec doesn't call for removing the reset handshake flag */
2701
2702         /*
2703          * Disable PW1 (PG1).
2704          * Note that even though the driver's request is removed power well 1
2705          * may stay enabled after this due to DMC's own request on it.
2706          */
2707         mutex_lock(&power_domains->lock);
2708
2709         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2710         intel_power_well_disable(dev_priv, well);
2711
2712         mutex_unlock(&power_domains->lock);
2713
2714         usleep_range(10, 30);           /* 10 us delay per Bspec */
2715 }
2716
2717 enum {
2718         PROCMON_0_85V_DOT_0,
2719         PROCMON_0_95V_DOT_0,
2720         PROCMON_0_95V_DOT_1,
2721         PROCMON_1_05V_DOT_0,
2722         PROCMON_1_05V_DOT_1,
2723 };
2724
2725 static const struct cnl_procmon {
2726         u32 dw1, dw9, dw10;
2727 } cnl_procmon_values[] = {
2728         [PROCMON_0_85V_DOT_0] =
2729                 { .dw1 = 0x00000000, .dw9 = 0x62AB67BB, .dw10 = 0x51914F96, },
2730         [PROCMON_0_95V_DOT_0] =
2731                 { .dw1 = 0x00000000, .dw9 = 0x86E172C7, .dw10 = 0x77CA5EAB, },
2732         [PROCMON_0_95V_DOT_1] =
2733                 { .dw1 = 0x00000000, .dw9 = 0x93F87FE1, .dw10 = 0x8AE871C5, },
2734         [PROCMON_1_05V_DOT_0] =
2735                 { .dw1 = 0x00000000, .dw9 = 0x98FA82DD, .dw10 = 0x89E46DC1, },
2736         [PROCMON_1_05V_DOT_1] =
2737                 { .dw1 = 0x00440000, .dw9 = 0x9A00AB25, .dw10 = 0x8AE38FF1, },
2738 };
2739
2740 static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv)
2741 {
2742         const struct cnl_procmon *procmon;
2743         u32 val;
2744
2745         val = I915_READ(CNL_PORT_COMP_DW3);
2746         switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) {
2747         default:
2748                 MISSING_CASE(val);
2749         case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0:
2750                 procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0];
2751                 break;
2752         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_0:
2753                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_0];
2754                 break;
2755         case VOLTAGE_INFO_0_95V | PROCESS_INFO_DOT_1:
2756                 procmon = &cnl_procmon_values[PROCMON_0_95V_DOT_1];
2757                 break;
2758         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_0:
2759                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_0];
2760                 break;
2761         case VOLTAGE_INFO_1_05V | PROCESS_INFO_DOT_1:
2762                 procmon = &cnl_procmon_values[PROCMON_1_05V_DOT_1];
2763                 break;
2764         }
2765
2766         val = I915_READ(CNL_PORT_COMP_DW1);
2767         val &= ~((0xff << 16) | 0xff);
2768         val |= procmon->dw1;
2769         I915_WRITE(CNL_PORT_COMP_DW1, val);
2770
2771         I915_WRITE(CNL_PORT_COMP_DW9, procmon->dw9);
2772         I915_WRITE(CNL_PORT_COMP_DW10, procmon->dw10);
2773 }
2774
2775 static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume)
2776 {
2777         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2778         struct i915_power_well *well;
2779         u32 val;
2780
2781         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2782
2783         /* 1. Enable PCH Reset Handshake */
2784         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2785         val |= RESET_PCH_HANDSHAKE_ENABLE;
2786         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2787
2788         /* 2. Enable Comp */
2789         val = I915_READ(CHICKEN_MISC_2);
2790         val &= ~CNL_COMP_PWR_DOWN;
2791         I915_WRITE(CHICKEN_MISC_2, val);
2792
2793         cnl_set_procmon_ref_values(dev_priv);
2794
2795         val = I915_READ(CNL_PORT_COMP_DW0);
2796         val |= COMP_INIT;
2797         I915_WRITE(CNL_PORT_COMP_DW0, val);
2798
2799         /* 3. */
2800         val = I915_READ(CNL_PORT_CL1CM_DW5);
2801         val |= CL_POWER_DOWN_ENABLE;
2802         I915_WRITE(CNL_PORT_CL1CM_DW5, val);
2803
2804         /*
2805          * 4. Enable Power Well 1 (PG1).
2806          *    The AUX IO power wells will be enabled on demand.
2807          */
2808         mutex_lock(&power_domains->lock);
2809         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2810         intel_power_well_enable(dev_priv, well);
2811         mutex_unlock(&power_domains->lock);
2812
2813         /* 5. Enable CD clock */
2814         cnl_init_cdclk(dev_priv);
2815
2816         /* 6. Enable DBUF */
2817         gen9_dbuf_enable(dev_priv);
2818
2819         if (resume && dev_priv->csr.dmc_payload)
2820                 intel_csr_load_program(dev_priv);
2821 }
2822
2823 static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
2824 {
2825         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2826         struct i915_power_well *well;
2827         u32 val;
2828
2829         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2830
2831         /* 1. Disable all display engine functions -> aready done */
2832
2833         /* 2. Disable DBUF */
2834         gen9_dbuf_disable(dev_priv);
2835
2836         /* 3. Disable CD clock */
2837         cnl_uninit_cdclk(dev_priv);
2838
2839         /*
2840          * 4. Disable Power Well 1 (PG1).
2841          *    The AUX IO power wells are toggled on demand, so they are already
2842          *    disabled at this point.
2843          */
2844         mutex_lock(&power_domains->lock);
2845         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2846         intel_power_well_disable(dev_priv, well);
2847         mutex_unlock(&power_domains->lock);
2848
2849         usleep_range(10, 30);           /* 10 us delay per Bspec */
2850
2851         /* 5. Disable Comp */
2852         val = I915_READ(CHICKEN_MISC_2);
2853         val |= CNL_COMP_PWR_DOWN;
2854         I915_WRITE(CHICKEN_MISC_2, val);
2855 }
2856
2857 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
2858 {
2859         struct i915_power_well *cmn_bc =
2860                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2861         struct i915_power_well *cmn_d =
2862                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
2863
2864         /*
2865          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
2866          * workaround never ever read DISPLAY_PHY_CONTROL, and
2867          * instead maintain a shadow copy ourselves. Use the actual
2868          * power well state and lane status to reconstruct the
2869          * expected initial value.
2870          */
2871         dev_priv->chv_phy_control =
2872                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
2873                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
2874                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
2875                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
2876                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
2877
2878         /*
2879          * If all lanes are disabled we leave the override disabled
2880          * with all power down bits cleared to match the state we
2881          * would use after disabling the port. Otherwise enable the
2882          * override and set the lane powerdown bits accding to the
2883          * current lane status.
2884          */
2885         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
2886                 uint32_t status = I915_READ(DPLL(PIPE_A));
2887                 unsigned int mask;
2888
2889                 mask = status & DPLL_PORTB_READY_MASK;
2890                 if (mask == 0xf)
2891                         mask = 0x0;
2892                 else
2893                         dev_priv->chv_phy_control |=
2894                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
2895
2896                 dev_priv->chv_phy_control |=
2897                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
2898
2899                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
2900                 if (mask == 0xf)
2901                         mask = 0x0;
2902                 else
2903                         dev_priv->chv_phy_control |=
2904                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
2905
2906                 dev_priv->chv_phy_control |=
2907                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
2908
2909                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
2910
2911                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
2912         } else {
2913                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
2914         }
2915
2916         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
2917                 uint32_t status = I915_READ(DPIO_PHY_STATUS);
2918                 unsigned int mask;
2919
2920                 mask = status & DPLL_PORTD_READY_MASK;
2921
2922                 if (mask == 0xf)
2923                         mask = 0x0;
2924                 else
2925                         dev_priv->chv_phy_control |=
2926                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
2927
2928                 dev_priv->chv_phy_control |=
2929                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
2930
2931                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
2932
2933                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
2934         } else {
2935                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
2936         }
2937
2938         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
2939
2940         DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
2941                       dev_priv->chv_phy_control);
2942 }
2943
2944 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
2945 {
2946         struct i915_power_well *cmn =
2947                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2948         struct i915_power_well *disp2d =
2949                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
2950
2951         /* If the display might be already active skip this */
2952         if (cmn->ops->is_enabled(dev_priv, cmn) &&
2953             disp2d->ops->is_enabled(dev_priv, disp2d) &&
2954             I915_READ(DPIO_CTL) & DPIO_CMNRST)
2955                 return;
2956
2957         DRM_DEBUG_KMS("toggling display PHY side reset\n");
2958
2959         /* cmnlane needs DPLL registers */
2960         disp2d->ops->enable(dev_priv, disp2d);
2961
2962         /*
2963          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
2964          * Need to assert and de-assert PHY SB reset by gating the
2965          * common lane power, then un-gating it.
2966          * Simply ungating isn't enough to reset the PHY enough to get
2967          * ports and lanes running.
2968          */
2969         cmn->ops->disable(dev_priv, cmn);
2970 }
2971
2972 /**
2973  * intel_power_domains_init_hw - initialize hardware power domain state
2974  * @dev_priv: i915 device instance
2975  * @resume: Called from resume code paths or not
2976  *
2977  * This function initializes the hardware power domain state and enables all
2978  * power wells belonging to the INIT power domain. Power wells in other
2979  * domains (and not in the INIT domain) are referenced or disabled during the
2980  * modeset state HW readout. After that the reference count of each power well
2981  * must match its HW enabled state, see intel_power_domains_verify_state().
2982  */
2983 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
2984 {
2985         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2986
2987         power_domains->initializing = true;
2988
2989         if (IS_CANNONLAKE(dev_priv)) {
2990                 cnl_display_core_init(dev_priv, resume);
2991         } else if (IS_GEN9_BC(dev_priv)) {
2992                 skl_display_core_init(dev_priv, resume);
2993         } else if (IS_GEN9_LP(dev_priv)) {
2994                 bxt_display_core_init(dev_priv, resume);
2995         } else if (IS_CHERRYVIEW(dev_priv)) {
2996                 mutex_lock(&power_domains->lock);
2997                 chv_phy_control_init(dev_priv);
2998                 mutex_unlock(&power_domains->lock);
2999         } else if (IS_VALLEYVIEW(dev_priv)) {
3000                 mutex_lock(&power_domains->lock);
3001                 vlv_cmnlane_wa(dev_priv);
3002                 mutex_unlock(&power_domains->lock);
3003         }
3004
3005         /* For now, we need the power well to be always enabled. */
3006         intel_display_set_init_power(dev_priv, true);
3007         /* Disable power support if the user asked so. */
3008         if (!i915_modparams.disable_power_well)
3009                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
3010         intel_power_domains_sync_hw(dev_priv);
3011         power_domains->initializing = false;
3012 }
3013
3014 /**
3015  * intel_power_domains_suspend - suspend power domain state
3016  * @dev_priv: i915 device instance
3017  *
3018  * This function prepares the hardware power domain state before entering
3019  * system suspend. It must be paired with intel_power_domains_init_hw().
3020  */
3021 void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
3022 {
3023         /*
3024          * Even if power well support was disabled we still want to disable
3025          * power wells while we are system suspended.
3026          */
3027         if (!i915_modparams.disable_power_well)
3028                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
3029
3030         if (IS_CANNONLAKE(dev_priv))
3031                 cnl_display_core_uninit(dev_priv);
3032         else if (IS_GEN9_BC(dev_priv))
3033                 skl_display_core_uninit(dev_priv);
3034         else if (IS_GEN9_LP(dev_priv))
3035                 bxt_display_core_uninit(dev_priv);
3036 }
3037
3038 static void intel_power_domains_dump_info(struct drm_i915_private *dev_priv)
3039 {
3040         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3041         struct i915_power_well *power_well;
3042
3043         for_each_power_well(dev_priv, power_well) {
3044                 enum intel_display_power_domain domain;
3045
3046                 DRM_DEBUG_DRIVER("%-25s %d\n",
3047                                  power_well->name, power_well->count);
3048
3049                 for_each_power_domain(domain, power_well->domains)
3050                         DRM_DEBUG_DRIVER("  %-23s %d\n",
3051                                          intel_display_power_domain_str(domain),
3052                                          power_domains->domain_use_count[domain]);
3053         }
3054 }
3055
3056 /**
3057  * intel_power_domains_verify_state - verify the HW/SW state for all power wells
3058  * @dev_priv: i915 device instance
3059  *
3060  * Verify if the reference count of each power well matches its HW enabled
3061  * state and the total refcount of the domains it belongs to. This must be
3062  * called after modeset HW state sanitization, which is responsible for
3063  * acquiring reference counts for any power wells in use and disabling the
3064  * ones left on by BIOS but not required by any active output.
3065  */
3066 void intel_power_domains_verify_state(struct drm_i915_private *dev_priv)
3067 {
3068         struct i915_power_domains *power_domains = &dev_priv->power_domains;
3069         struct i915_power_well *power_well;
3070         bool dump_domain_info;
3071
3072         mutex_lock(&power_domains->lock);
3073
3074         dump_domain_info = false;
3075         for_each_power_well(dev_priv, power_well) {
3076                 enum intel_display_power_domain domain;
3077                 int domains_count;
3078                 bool enabled;
3079
3080                 /*
3081                  * Power wells not belonging to any domain (like the MISC_IO
3082                  * and PW1 power wells) are under FW control, so ignore them,
3083                  * since their state can change asynchronously.
3084                  */
3085                 if (!power_well->domains)
3086                         continue;
3087
3088                 enabled = power_well->ops->is_enabled(dev_priv, power_well);
3089                 if ((power_well->count || power_well->always_on) != enabled)
3090                         DRM_ERROR("power well %s state mismatch (refcount %d/enabled %d)",
3091                                   power_well->name, power_well->count, enabled);
3092
3093                 domains_count = 0;
3094                 for_each_power_domain(domain, power_well->domains)
3095                         domains_count += power_domains->domain_use_count[domain];
3096
3097                 if (power_well->count != domains_count) {
3098                         DRM_ERROR("power well %s refcount/domain refcount mismatch "
3099                                   "(refcount %d/domains refcount %d)\n",
3100                                   power_well->name, power_well->count,
3101                                   domains_count);
3102                         dump_domain_info = true;
3103                 }
3104         }
3105
3106         if (dump_domain_info) {
3107                 static bool dumped;
3108
3109                 if (!dumped) {
3110                         intel_power_domains_dump_info(dev_priv);
3111                         dumped = true;
3112                 }
3113         }
3114
3115         mutex_unlock(&power_domains->lock);
3116 }
3117
3118 /**
3119  * intel_runtime_pm_get - grab a runtime pm reference
3120  * @dev_priv: i915 device instance
3121  *
3122  * This function grabs a device-level runtime pm reference (mostly used for GEM
3123  * code to ensure the GTT or GT is on) and ensures that it is powered up.
3124  *
3125  * Any runtime pm reference obtained by this function must have a symmetric
3126  * call to intel_runtime_pm_put() to release the reference again.
3127  */
3128 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
3129 {
3130         struct pci_dev *pdev = dev_priv->drm.pdev;
3131         struct device *kdev = &pdev->dev;
3132         int ret;
3133
3134         ret = pm_runtime_get_sync(kdev);
3135         WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3136
3137         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3138         assert_rpm_wakelock_held(dev_priv);
3139 }
3140
3141 /**
3142  * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
3143  * @dev_priv: i915 device instance
3144  *
3145  * This function grabs a device-level runtime pm reference if the device is
3146  * already in use and ensures that it is powered up.
3147  *
3148  * Any runtime pm reference obtained by this function must have a symmetric
3149  * call to intel_runtime_pm_put() to release the reference again.
3150  */
3151 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
3152 {
3153         struct pci_dev *pdev = dev_priv->drm.pdev;
3154         struct device *kdev = &pdev->dev;
3155
3156         if (IS_ENABLED(CONFIG_PM)) {
3157                 int ret = pm_runtime_get_if_in_use(kdev);
3158
3159                 /*
3160                  * In cases runtime PM is disabled by the RPM core and we get
3161                  * an -EINVAL return value we are not supposed to call this
3162                  * function, since the power state is undefined. This applies
3163                  * atm to the late/early system suspend/resume handlers.
3164                  */
3165                 WARN_ONCE(ret < 0,
3166                           "pm_runtime_get_if_in_use() failed: %d\n", ret);
3167                 if (ret <= 0)
3168                         return false;
3169         }
3170
3171         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3172         assert_rpm_wakelock_held(dev_priv);
3173
3174         return true;
3175 }
3176
3177 /**
3178  * intel_runtime_pm_get_noresume - grab a runtime pm reference
3179  * @dev_priv: i915 device instance
3180  *
3181  * This function grabs a device-level runtime pm reference (mostly used for GEM
3182  * code to ensure the GTT or GT is on).
3183  *
3184  * It will _not_ power up the device but instead only check that it's powered
3185  * on.  Therefore it is only valid to call this functions from contexts where
3186  * the device is known to be powered up and where trying to power it up would
3187  * result in hilarity and deadlocks. That pretty much means only the system
3188  * suspend/resume code where this is used to grab runtime pm references for
3189  * delayed setup down in work items.
3190  *
3191  * Any runtime pm reference obtained by this function must have a symmetric
3192  * call to intel_runtime_pm_put() to release the reference again.
3193  */
3194 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
3195 {
3196         struct pci_dev *pdev = dev_priv->drm.pdev;
3197         struct device *kdev = &pdev->dev;
3198
3199         assert_rpm_wakelock_held(dev_priv);
3200         pm_runtime_get_noresume(kdev);
3201
3202         atomic_inc(&dev_priv->runtime_pm.wakeref_count);
3203 }
3204
3205 /**
3206  * intel_runtime_pm_put - release a runtime pm reference
3207  * @dev_priv: i915 device instance
3208  *
3209  * This function drops the device-level runtime pm reference obtained by
3210  * intel_runtime_pm_get() and might power down the corresponding
3211  * hardware block right away if this is the last reference.
3212  */
3213 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
3214 {
3215         struct pci_dev *pdev = dev_priv->drm.pdev;
3216         struct device *kdev = &pdev->dev;
3217
3218         assert_rpm_wakelock_held(dev_priv);
3219         atomic_dec(&dev_priv->runtime_pm.wakeref_count);
3220
3221         pm_runtime_mark_last_busy(kdev);
3222         pm_runtime_put_autosuspend(kdev);
3223 }
3224
3225 /**
3226  * intel_runtime_pm_enable - enable runtime pm
3227  * @dev_priv: i915 device instance
3228  *
3229  * This function enables runtime pm at the end of the driver load sequence.
3230  *
3231  * Note that this function does currently not enable runtime pm for the
3232  * subordinate display power domains. That is only done on the first modeset
3233  * using intel_display_set_init_power().
3234  */
3235 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
3236 {
3237         struct pci_dev *pdev = dev_priv->drm.pdev;
3238         struct device *kdev = &pdev->dev;
3239
3240         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
3241         pm_runtime_mark_last_busy(kdev);
3242
3243         /*
3244          * Take a permanent reference to disable the RPM functionality and drop
3245          * it only when unloading the driver. Use the low level get/put helpers,
3246          * so the driver's own RPM reference tracking asserts also work on
3247          * platforms without RPM support.
3248          */
3249         if (!HAS_RUNTIME_PM(dev_priv)) {
3250                 int ret;
3251
3252                 pm_runtime_dont_use_autosuspend(kdev);
3253                 ret = pm_runtime_get_sync(kdev);
3254                 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret);
3255         } else {
3256                 pm_runtime_use_autosuspend(kdev);
3257         }
3258
3259         /*
3260          * The core calls the driver load handler with an RPM reference held.
3261          * We drop that here and will reacquire it during unloading in
3262          * intel_power_domains_fini().
3263          */
3264         pm_runtime_put_autosuspend(kdev);
3265 }