drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include <drm/drm_plane_helper.h>
  30 #include "i915_drv.h"
  31 #include "intel_drv.h"
  32 #include "../../../platform/x86/intel_ips.h"
  33 #include <linux/module.h>
  34 #include <drm/drm_atomic_helper.h>
  35
  36 /**
  37  * DOC: RC6
  38  *
  39  * RC6 is a special power stage which allows the GPU to enter an very
  40  * low-voltage mode when idle, using down to 0V while at this stage.  This
  41  * stage is entered automatically when the GPU is idle when RC6 support is
  42  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  43  *
  44  * There are different RC6 modes available in Intel GPU, which differentiate
  45  * among each other with the latency required to enter and leave RC6 and
  46  * voltage consumed by the GPU in different states.
  47  *
  48  * The combination of the following flags define which states GPU is allowed
  49  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  50  * RC6pp is deepest RC6. Their support by hardware varies according to the
  51  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  52  * which brings the most power savings; deeper states save more power, but
  53  * require higher latency to switch to and wake up.
  54  */
  55
  56 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  57 {
  58         if (HAS_LLC(dev_priv)) {
  59                 /*
  60                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  61                  * Display WA #0390: skl,kbl
  62                  *
  63                  * Must match Sampler, Pixel Back End, and Media. See
  64                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  65                  */
  66                 I915_WRITE(CHICKEN_PAR1_1,
  67                            I915_READ(CHICKEN_PAR1_1) |
  68                            SKL_DE_COMPRESSED_HASH_MODE);
  69         }
  70
  71         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  72         I915_WRITE(CHICKEN_PAR1_1,
  73                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  74
  75         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  76         I915_WRITE(GEN8_CHICKEN_DCPR_1,
  77                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  78
  79         /* WaFbcTurnOffFbcWatermark:skl,bxt,kbl,cfl */
  80         /* WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl */
  81         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
  82                    DISP_FBC_WM_DIS |
  83                    DISP_FBC_MEMORY_WAKE);
  84
  85         /* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
  86         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
  87                    ILK_DPFC_DISABLE_DUMMY0);
  88
  89         if (IS_SKYLAKE(dev_priv)) {
  90                 /* WaDisableDopClockGating */
  91                 I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
  92                            & ~GEN7_DOP_CLOCK_GATE_ENABLE);
  93         }
  94 }
  95
  96 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  97 {
  98         gen9_init_clock_gating(dev_priv);
  99
 100         /* WaDisableSDEUnitClockGating:bxt */
 101         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 102                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 103
 104         /*
 105          * FIXME:
 106          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
 107          */
 108         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 109                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 110
 111         /*
 112          * Wa: Backlight PWM may stop in the asserted state, causing backlight
 113          * to stay fully on.
 114          */
 115         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 116                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 117 }
 118
 119 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 120 {
 121         gen9_init_clock_gating(dev_priv);
 122
 123         /*
 124          * WaDisablePWMClockGating:glk
 125          * Backlight PWM may stop in the asserted state, causing backlight
 126          * to stay fully on.
 127          */
 128         I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
 129                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 130
 131         /* WaDDIIOTimeout:glk */
 132         if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) {
 133                 u32 val = I915_READ(CHICKEN_MISC_2);
 134                 val &= ~(GLK_CL0_PWR_DOWN |
 135                          GLK_CL1_PWR_DOWN |
 136                          GLK_CL2_PWR_DOWN);
 137                 I915_WRITE(CHICKEN_MISC_2, val);
 138         }
 139
 140 }
 141
 142 static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
 143 {
 144         u32 tmp;
 145
 146         tmp = I915_READ(CLKCFG);
 147
 148         switch (tmp & CLKCFG_FSB_MASK) {
 149         case CLKCFG_FSB_533:
 150                 dev_priv->fsb_freq = 533; /* 133*4 */
 151                 break;
 152         case CLKCFG_FSB_800:
 153                 dev_priv->fsb_freq = 800; /* 200*4 */
 154                 break;
 155         case CLKCFG_FSB_667:
 156                 dev_priv->fsb_freq =  667; /* 167*4 */
 157                 break;
 158         case CLKCFG_FSB_400:
 159                 dev_priv->fsb_freq = 400; /* 100*4 */
 160                 break;
 161         }
 162
 163         switch (tmp & CLKCFG_MEM_MASK) {
 164         case CLKCFG_MEM_533:
 165                 dev_priv->mem_freq = 533;
 166                 break;
 167         case CLKCFG_MEM_667:
 168                 dev_priv->mem_freq = 667;
 169                 break;
 170         case CLKCFG_MEM_800:
 171                 dev_priv->mem_freq = 800;
 172                 break;
 173         }
 174
 175         /* detect pineview DDR3 setting */
 176         tmp = I915_READ(CSHRDDR3CTL);
 177         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 178 }
 179
 180 static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv)
 181 {
 182         u16 ddrpll, csipll;
 183
 184         ddrpll = I915_READ16(DDRMPLL1);
 185         csipll = I915_READ16(CSIPLL0);
 186
 187         switch (ddrpll & 0xff) {
 188         case 0xc:
 189                 dev_priv->mem_freq = 800;
 190                 break;
 191         case 0x10:
 192                 dev_priv->mem_freq = 1066;
 193                 break;
 194         case 0x14:
 195                 dev_priv->mem_freq = 1333;
 196                 break;
 197         case 0x18:
 198                 dev_priv->mem_freq = 1600;
 199                 break;
 200         default:
 201                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 202                                  ddrpll & 0xff);
 203                 dev_priv->mem_freq = 0;
 204                 break;
 205         }
 206
 207         dev_priv->ips.r_t = dev_priv->mem_freq;
 208
 209         switch (csipll & 0x3ff) {
 210         case 0x00c:
 211                 dev_priv->fsb_freq = 3200;
 212                 break;
 213         case 0x00e:
 214                 dev_priv->fsb_freq = 3733;
 215                 break;
 216         case 0x010:
 217                 dev_priv->fsb_freq = 4266;
 218                 break;
 219         case 0x012:
 220                 dev_priv->fsb_freq = 4800;
 221                 break;
 222         case 0x014:
 223                 dev_priv->fsb_freq = 5333;
 224                 break;
 225         case 0x016:
 226                 dev_priv->fsb_freq = 5866;
 227                 break;
 228         case 0x018:
 229                 dev_priv->fsb_freq = 6400;
 230                 break;
 231         default:
 232                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 233                                  csipll & 0x3ff);
 234                 dev_priv->fsb_freq = 0;
 235                 break;
 236         }
 237
 238         if (dev_priv->fsb_freq == 3200) {
 239                 dev_priv->ips.c_m = 0;
 240         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 241                 dev_priv->ips.c_m = 1;
 242         } else {
 243                 dev_priv->ips.c_m = 2;
 244         }
 245 }
 246
 247 static const struct cxsr_latency cxsr_latency_table[] = {
 248         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 249         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 250         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 251         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 252         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 253
 254         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 255         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 256         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 257         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 258         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 259
 260         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 261         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 262         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 263         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 264         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 265
 266         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 267         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 268         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 269         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 270         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 271
 272         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 273         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 274         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 275         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 276         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 277
 278         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 279         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 280         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 281         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 282         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 283 };
 284
 285 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 286                                                          bool is_ddr3,
 287                                                          int fsb,
 288                                                          int mem)
 289 {
 290         const struct cxsr_latency *latency;
 291         int i;
 292
 293         if (fsb == 0 || mem == 0)
 294                 return NULL;
 295
 296         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 297                 latency = &cxsr_latency_table[i];
 298                 if (is_desktop == latency->is_desktop &&
 299                     is_ddr3 == latency->is_ddr3 &&
 300                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 301                         return latency;
 302         }
 303
 304         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 305
 306         return NULL;
 307 }
 308
 309 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 310 {
 311         u32 val;
 312
 313         mutex_lock(&dev_priv->pcu_lock);
 314
 315         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 316         if (enable)
 317                 val &= ~FORCE_DDR_HIGH_FREQ;
 318         else
 319                 val |= FORCE_DDR_HIGH_FREQ;
 320         val &= ~FORCE_DDR_LOW_FREQ;
 321         val |= FORCE_DDR_FREQ_REQ_ACK;
 322         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 323
 324         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 325                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 326                 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
 327
 328         mutex_unlock(&dev_priv->pcu_lock);
 329 }
 330
 331 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 332 {
 333         u32 val;
 334
 335         mutex_lock(&dev_priv->pcu_lock);
 336
 337         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 338         if (enable)
 339                 val |= DSP_MAXFIFO_PM5_ENABLE;
 340         else
 341                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 342         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
 343
 344         mutex_unlock(&dev_priv->pcu_lock);
 345 }
 346
 347 #define FW_WM(value, plane) \
 348         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 349
 350 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 351 {
 352         bool was_enabled;
 353         u32 val;
 354
 355         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 356                 was_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 357                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 358                 POSTING_READ(FW_BLC_SELF_VLV);
 359         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 360                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 361                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 362                 POSTING_READ(FW_BLC_SELF);
 363         } else if (IS_PINEVIEW(dev_priv)) {
 364                 val = I915_READ(DSPFW3);
 365                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 366                 if (enable)
 367                         val |= PINEVIEW_SELF_REFRESH_EN;
 368                 else
 369                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 370                 I915_WRITE(DSPFW3, val);
 371                 POSTING_READ(DSPFW3);
 372         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 373                 was_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
 374                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 375                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 376                 I915_WRITE(FW_BLC_SELF, val);
 377                 POSTING_READ(FW_BLC_SELF);
 378         } else if (IS_I915GM(dev_priv)) {
 379                 /*
 380                  * FIXME can't find a bit like this for 915G, and
 381                  * and yet it does have the related watermark in
 382                  * FW_BLC_SELF. What's going on?
 383                  */
 384                 was_enabled = I915_READ(INSTPM) & INSTPM_SELF_EN;
 385                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 386                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 387                 I915_WRITE(INSTPM, val);
 388                 POSTING_READ(INSTPM);
 389         } else {
 390                 return false;
 391         }
 392
 393         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 394
 395         DRM_DEBUG_KMS("memory self-refresh is %s (was %s)\n",
 396                       enableddisabled(enable),
 397                       enableddisabled(was_enabled));
 398
 399         return was_enabled;
 400 }
 401
 402 /**
 403  * intel_set_memory_cxsr - Configure CxSR state
 404  * @dev_priv: i915 device
 405  * @enable: Allow vs. disallow CxSR
 406  *
 407  * Allow or disallow the system to enter a special CxSR
 408  * (C-state self refresh) state. What typically happens in CxSR mode
 409  * is that several display FIFOs may get combined into a single larger
 410  * FIFO for a particular plane (so called max FIFO mode) to allow the
 411  * system to defer memory fetches longer, and the memory will enter
 412  * self refresh.
 413  *
 414  * Note that enabling CxSR does not guarantee that the system enter
 415  * this special mode, nor does it guarantee that the system stays
 416  * in that mode once entered. So this just allows/disallows the system
 417  * to autonomously utilize the CxSR mode. Other factors such as core
 418  * C-states will affect when/if the system actually enters/exits the
 419  * CxSR mode.
 420  *
 421  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 422  * and the system is free to enter/exit memory self refresh at any time
 423  * even when the use of CxSR has been disallowed.
 424  *
 425  * While the system is actually in the CxSR/max FIFO mode, some plane
 426  * control registers will not get latched on vblank. Thus in order to
 427  * guarantee the system will respond to changes in the plane registers
 428  * we must always disallow CxSR prior to making changes to those registers.
 429  * Unfortunately the system will re-evaluate the CxSR conditions at
 430  * frame start which happens after vblank start (which is when the plane
 431  * registers would get latched), so we can't proceed with the plane update
 432  * during the same frame where we disallowed CxSR.
 433  *
 434  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 435  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 436  * the hardware w.r.t. HPLL SR when writing to plane registers.
 437  * Disallowing just CxSR is sufficient.
 438  */
 439 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 440 {
 441         bool ret;
 442
 443         mutex_lock(&dev_priv->wm.wm_mutex);
 444         ret = _intel_set_memory_cxsr(dev_priv, enable);
 445         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 446                 dev_priv->wm.vlv.cxsr = enable;
 447         else if (IS_G4X(dev_priv))
 448                 dev_priv->wm.g4x.cxsr = enable;
 449         mutex_unlock(&dev_priv->wm.wm_mutex);
 450
 451         return ret;
 452 }
 453
 454 /*
 455  * Latency for FIFO fetches is dependent on several factors:
 456  *   - memory configuration (speed, channels)
 457  *   - chipset
 458  *   - current MCH state
 459  * It can be fairly high in some situations, so here we assume a fairly
 460  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 461  * set this value too high, the FIFO will fetch frequently to stay full)
 462  * and power consumption (set it too low to save power and we might see
 463  * FIFO underruns and display "flicker").
 464  *
 465  * A value of 5us seems to be a good balance; safe for very low end
 466  * platforms but not overly aggressive on lower latency configs.
 467  */
 468 static const int pessimal_latency_ns = 5000;
 469
 470 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 471         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 472
 473 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 474 {
 475         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 476         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 477         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 478         enum pipe pipe = crtc->pipe;
 479         int sprite0_start, sprite1_start;
 480
 481         switch (pipe) {
 482                 uint32_t dsparb, dsparb2, dsparb3;
 483         case PIPE_A:
 484                 dsparb = I915_READ(DSPARB);
 485                 dsparb2 = I915_READ(DSPARB2);
 486                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 487                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 488                 break;
 489         case PIPE_B:
 490                 dsparb = I915_READ(DSPARB);
 491                 dsparb2 = I915_READ(DSPARB2);
 492                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 493                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 494                 break;
 495         case PIPE_C:
 496                 dsparb2 = I915_READ(DSPARB2);
 497                 dsparb3 = I915_READ(DSPARB3);
 498                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 499                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 500                 break;
 501         default:
 502                 MISSING_CASE(pipe);
 503                 return;
 504         }
 505
 506         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 507         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 508         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 509         fifo_state->plane[PLANE_CURSOR] = 63;
 510 }
 511
 512 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 513                               enum i9xx_plane_id i9xx_plane)
 514 {
 515         uint32_t dsparb = I915_READ(DSPARB);
 516         int size;
 517
 518         size = dsparb & 0x7f;
 519         if (i9xx_plane == PLANE_B)
 520                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 521
 522         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 523                       dsparb, plane_name(i9xx_plane), size);
 524
 525         return size;
 526 }
 527
 528 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 529                               enum i9xx_plane_id i9xx_plane)
 530 {
 531         uint32_t dsparb = I915_READ(DSPARB);
 532         int size;
 533
 534         size = dsparb & 0x1ff;
 535         if (i9xx_plane == PLANE_B)
 536                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 537         size >>= 1; /* Convert to cachelines */
 538
 539         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 540                       dsparb, plane_name(i9xx_plane), size);
 541
 542         return size;
 543 }
 544
 545 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 546                               enum i9xx_plane_id i9xx_plane)
 547 {
 548         uint32_t dsparb = I915_READ(DSPARB);
 549         int size;
 550
 551         size = dsparb & 0x7f;
 552         size >>= 2; /* Convert to cachelines */
 553
 554         DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
 555                       dsparb, plane_name(i9xx_plane), size);
 556
 557         return size;
 558 }
 559
 560 /* Pineview has different values for various configs */
 561 static const struct intel_watermark_params pineview_display_wm = {
 562         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 563         .max_wm = PINEVIEW_MAX_WM,
 564         .default_wm = PINEVIEW_DFT_WM,
 565         .guard_size = PINEVIEW_GUARD_WM,
 566         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 567 };
 568 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 569         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 570         .max_wm = PINEVIEW_MAX_WM,
 571         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 572         .guard_size = PINEVIEW_GUARD_WM,
 573         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 574 };
 575 static const struct intel_watermark_params pineview_cursor_wm = {
 576         .fifo_size = PINEVIEW_CURSOR_FIFO,
 577         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 578         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 579         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 580         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 581 };
 582 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 583         .fifo_size = PINEVIEW_CURSOR_FIFO,
 584         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 585         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 586         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 587         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 588 };
 589 static const struct intel_watermark_params i965_cursor_wm_info = {
 590         .fifo_size = I965_CURSOR_FIFO,
 591         .max_wm = I965_CURSOR_MAX_WM,
 592         .default_wm = I965_CURSOR_DFT_WM,
 593         .guard_size = 2,
 594         .cacheline_size = I915_FIFO_LINE_SIZE,
 595 };
 596 static const struct intel_watermark_params i945_wm_info = {
 597         .fifo_size = I945_FIFO_SIZE,
 598         .max_wm = I915_MAX_WM,
 599         .default_wm = 1,
 600         .guard_size = 2,
 601         .cacheline_size = I915_FIFO_LINE_SIZE,
 602 };
 603 static const struct intel_watermark_params i915_wm_info = {
 604         .fifo_size = I915_FIFO_SIZE,
 605         .max_wm = I915_MAX_WM,
 606         .default_wm = 1,
 607         .guard_size = 2,
 608         .cacheline_size = I915_FIFO_LINE_SIZE,
 609 };
 610 static const struct intel_watermark_params i830_a_wm_info = {
 611         .fifo_size = I855GM_FIFO_SIZE,
 612         .max_wm = I915_MAX_WM,
 613         .default_wm = 1,
 614         .guard_size = 2,
 615         .cacheline_size = I830_FIFO_LINE_SIZE,
 616 };
 617 static const struct intel_watermark_params i830_bc_wm_info = {
 618         .fifo_size = I855GM_FIFO_SIZE,
 619         .max_wm = I915_MAX_WM/2,
 620         .default_wm = 1,
 621         .guard_size = 2,
 622         .cacheline_size = I830_FIFO_LINE_SIZE,
 623 };
 624 static const struct intel_watermark_params i845_wm_info = {
 625         .fifo_size = I830_FIFO_SIZE,
 626         .max_wm = I915_MAX_WM,
 627         .default_wm = 1,
 628         .guard_size = 2,
 629         .cacheline_size = I830_FIFO_LINE_SIZE,
 630 };
 631
 632 /**
 633  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 634  * @pixel_rate: Pipe pixel rate in kHz
 635  * @cpp: Plane bytes per pixel
 636  * @latency: Memory wakeup latency in 0.1us units
 637  *
 638  * Compute the watermark using the method 1 or "small buffer"
 639  * formula. The caller may additonally add extra cachelines
 640  * to account for TLB misses and clock crossings.
 641  *
 642  * This method is concerned with the short term drain rate
 643  * of the FIFO, ie. it does not account for blanking periods
 644  * which would effectively reduce the average drain rate across
 645  * a longer period. The name "small" refers to the fact the
 646  * FIFO is relatively small compared to the amount of data
 647  * fetched.
 648  *
 649  * The FIFO level vs. time graph might look something like:
 650  *
 651  *   |\   |\
 652  *   | \  | \
 653  * __---__---__ (- plane active, _ blanking)
 654  * -> time
 655  *
 656  * or perhaps like this:
 657  *
 658  *   |\|\  |\|\
 659  * __----__----__ (- plane active, _ blanking)
 660  * -> time
 661  *
 662  * Returns:
 663  * The watermark in bytes
 664  */
 665 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 666                                      unsigned int cpp,
 667                                      unsigned int latency)
 668 {
 669         uint64_t ret;
 670
 671         ret = (uint64_t) pixel_rate * cpp * latency;
 672         ret = DIV_ROUND_UP_ULL(ret, 10000);
 673
 674         return ret;
 675 }
 676
 677 /**
 678  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 679  * @pixel_rate: Pipe pixel rate in kHz
 680  * @htotal: Pipe horizontal total
 681  * @width: Plane width in pixels
 682  * @cpp: Plane bytes per pixel
 683  * @latency: Memory wakeup latency in 0.1us units
 684  *
 685  * Compute the watermark using the method 2 or "large buffer"
 686  * formula. The caller may additonally add extra cachelines
 687  * to account for TLB misses and clock crossings.
 688  *
 689  * This method is concerned with the long term drain rate
 690  * of the FIFO, ie. it does account for blanking periods
 691  * which effectively reduce the average drain rate across
 692  * a longer period. The name "large" refers to the fact the
 693  * FIFO is relatively large compared to the amount of data
 694  * fetched.
 695  *
 696  * The FIFO level vs. time graph might look something like:
 697  *
 698  *    |\___       |\___
 699  *    |    \___   |    \___
 700  *    |        \  |        \
 701  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 702  * -> time
 703  *
 704  * Returns:
 705  * The watermark in bytes
 706  */
 707 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 708                                      unsigned int htotal,
 709                                      unsigned int width,
 710                                      unsigned int cpp,
 711                                      unsigned int latency)
 712 {
 713         unsigned int ret;
 714
 715         /*
 716          * FIXME remove once all users are computing
 717          * watermarks in the correct place.
 718          */
 719         if (WARN_ON_ONCE(htotal == 0))
 720                 htotal = 1;
 721
 722         ret = (latency * pixel_rate) / (htotal * 10000);
 723         ret = (ret + 1) * width * cpp;
 724
 725         return ret;
 726 }
 727
 728 /**
 729  * intel_calculate_wm - calculate watermark level
 730  * @pixel_rate: pixel clock
 731  * @wm: chip FIFO params
 732  * @cpp: bytes per pixel
 733  * @latency_ns: memory latency for the platform
 734  *
 735  * Calculate the watermark level (the level at which the display plane will
 736  * start fetching from memory again).  Each chip has a different display
 737  * FIFO size and allocation, so the caller needs to figure that out and pass
 738  * in the correct intel_watermark_params structure.
 739  *
 740  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 741  * on the pixel size.  When it reaches the watermark level, it'll start
 742  * fetching FIFO line sized based chunks from memory until the FIFO fills
 743  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 744  * will occur, and a display engine hang could result.
 745  */
 746 static unsigned int intel_calculate_wm(int pixel_rate,
 747                                        const struct intel_watermark_params *wm,
 748                                        int fifo_size, int cpp,
 749                                        unsigned int latency_ns)
 750 {
 751         int entries, wm_size;
 752
 753         /*
 754          * Note: we need to make sure we don't overflow for various clock &
 755          * latency values.
 756          * clocks go from a few thousand to several hundred thousand.
 757          * latency is usually a few thousand
 758          */
 759         entries = intel_wm_method1(pixel_rate, cpp,
 760                                    latency_ns / 100);
 761         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 762                 wm->guard_size;
 763         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 764
 765         wm_size = fifo_size - entries;
 766         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 767
 768         /* Don't promote wm_size to unsigned... */
 769         if (wm_size > wm->max_wm)
 770                 wm_size = wm->max_wm;
 771         if (wm_size <= 0)
 772                 wm_size = wm->default_wm;
 773
 774         /*
 775          * Bspec seems to indicate that the value shouldn't be lower than
 776          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 777          * Lets go for 8 which is the burst size since certain platforms
 778          * already use a hardcoded 8 (which is what the spec says should be
 779          * done).
 780          */
 781         if (wm_size <= 8)
 782                 wm_size = 8;
 783
 784         return wm_size;
 785 }
 786
 787 static bool is_disabling(int old, int new, int threshold)
 788 {
 789         return old >= threshold && new < threshold;
 790 }
 791
 792 static bool is_enabling(int old, int new, int threshold)
 793 {
 794         return old < threshold && new >= threshold;
 795 }
 796
 797 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 798 {
 799         return dev_priv->wm.max_level + 1;
 800 }
 801
 802 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 803                                    const struct intel_plane_state *plane_state)
 804 {
 805         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 806
 807         /* FIXME check the 'enable' instead */
 808         if (!crtc_state->base.active)
 809                 return false;
 810
 811         /*
 812          * Treat cursor with fb as always visible since cursor updates
 813          * can happen faster than the vrefresh rate, and the current
 814          * watermark code doesn't handle that correctly. Cursor updates
 815          * which set/clear the fb or change the cursor size are going
 816          * to get throttled by intel_legacy_cursor_update() to work
 817          * around this problem with the watermark code.
 818          */
 819         if (plane->id == PLANE_CURSOR)
 820                 return plane_state->base.fb != NULL;
 821         else
 822                 return plane_state->base.visible;
 823 }
 824
 825 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 826 {
 827         struct intel_crtc *crtc, *enabled = NULL;
 828
 829         for_each_intel_crtc(&dev_priv->drm, crtc) {
 830                 if (intel_crtc_active(crtc)) {
 831                         if (enabled)
 832                                 return NULL;
 833                         enabled = crtc;
 834                 }
 835         }
 836
 837         return enabled;
 838 }
 839
 840 static void pineview_update_wm(struct intel_crtc *unused_crtc)
 841 {
 842         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
 843         struct intel_crtc *crtc;
 844         const struct cxsr_latency *latency;
 845         u32 reg;
 846         unsigned int wm;
 847
 848         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
 849                                          dev_priv->is_ddr3,
 850                                          dev_priv->fsb_freq,
 851                                          dev_priv->mem_freq);
 852         if (!latency) {
 853                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 854                 intel_set_memory_cxsr(dev_priv, false);
 855                 return;
 856         }
 857
 858         crtc = single_enabled_crtc(dev_priv);
 859         if (crtc) {
 860                 const struct drm_display_mode *adjusted_mode =
 861                         &crtc->config->base.adjusted_mode;
 862                 const struct drm_framebuffer *fb =
 863                         crtc->base.primary->state->fb;
 864                 int cpp = fb->format->cpp[0];
 865                 int clock = adjusted_mode->crtc_clock;
 866
 867                 /* Display SR */
 868                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 869                                         pineview_display_wm.fifo_size,
 870                                         cpp, latency->display_sr);
 871                 reg = I915_READ(DSPFW1);
 872                 reg &= ~DSPFW_SR_MASK;
 873                 reg |= FW_WM(wm, SR);
 874                 I915_WRITE(DSPFW1, reg);
 875                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 876
 877                 /* cursor SR */
 878                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 879                                         pineview_display_wm.fifo_size,
 880                                         4, latency->cursor_sr);
 881                 reg = I915_READ(DSPFW3);
 882                 reg &= ~DSPFW_CURSOR_SR_MASK;
 883                 reg |= FW_WM(wm, CURSOR_SR);
 884                 I915_WRITE(DSPFW3, reg);
 885
 886                 /* Display HPLL off SR */
 887                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 888                                         pineview_display_hplloff_wm.fifo_size,
 889                                         cpp, latency->display_hpll_disable);
 890                 reg = I915_READ(DSPFW3);
 891                 reg &= ~DSPFW_HPLL_SR_MASK;
 892                 reg |= FW_WM(wm, HPLL_SR);
 893                 I915_WRITE(DSPFW3, reg);
 894
 895                 /* cursor HPLL off SR */
 896                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 897                                         pineview_display_hplloff_wm.fifo_size,
 898                                         4, latency->cursor_hpll_disable);
 899                 reg = I915_READ(DSPFW3);
 900                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 901                 reg |= FW_WM(wm, HPLL_CURSOR);
 902                 I915_WRITE(DSPFW3, reg);
 903                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 904
 905                 intel_set_memory_cxsr(dev_priv, true);
 906         } else {
 907                 intel_set_memory_cxsr(dev_priv, false);
 908         }
 909 }
 910
 911 /*
 912  * Documentation says:
 913  * "If the line size is small, the TLB fetches can get in the way of the
 914  *  data fetches, causing some lag in the pixel data return which is not
 915  *  accounted for in the above formulas. The following adjustment only
 916  *  needs to be applied if eight whole lines fit in the buffer at once.
 917  *  The WM is adjusted upwards by the difference between the FIFO size
 918  *  and the size of 8 whole lines. This adjustment is always performed
 919  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 920  */
 921 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 922 {
 923         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 924
 925         return max(0, tlb_miss);
 926 }
 927
 928 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 929                                 const struct g4x_wm_values *wm)
 930 {
 931         enum pipe pipe;
 932
 933         for_each_pipe(dev_priv, pipe)
 934                 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 935
 936         I915_WRITE(DSPFW1,
 937                    FW_WM(wm->sr.plane, SR) |
 938                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 939                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 940                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 941         I915_WRITE(DSPFW2,
 942                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 943                    FW_WM(wm->sr.fbc, FBC_SR) |
 944                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 945                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 946                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 947                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 948         I915_WRITE(DSPFW3,
 949                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 950                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 951                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 952                    FW_WM(wm->hpll.plane, HPLL_SR));
 953
 954         POSTING_READ(DSPFW1);
 955 }
 956
 957 #define FW_WM_VLV(value, plane) \
 958         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 959
 960 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 961                                 const struct vlv_wm_values *wm)
 962 {
 963         enum pipe pipe;
 964
 965         for_each_pipe(dev_priv, pipe) {
 966                 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
 967
 968                 I915_WRITE(VLV_DDL(pipe),
 969                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 970                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 971                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 972                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 973         }
 974
 975         /*
 976          * Zero the (unused) WM1 watermarks, and also clear all the
 977          * high order bits so that there are no out of bounds values
 978          * present in the registers during the reprogramming.
 979          */
 980         I915_WRITE(DSPHOWM, 0);
 981         I915_WRITE(DSPHOWM1, 0);
 982         I915_WRITE(DSPFW4, 0);
 983         I915_WRITE(DSPFW5, 0);
 984         I915_WRITE(DSPFW6, 0);
 985
 986         I915_WRITE(DSPFW1,
 987                    FW_WM(wm->sr.plane, SR) |
 988                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 989                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 990                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 991         I915_WRITE(DSPFW2,
 992                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
 993                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 994                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 995         I915_WRITE(DSPFW3,
 996                    FW_WM(wm->sr.cursor, CURSOR_SR));
 997
 998         if (IS_CHERRYVIEW(dev_priv)) {
 999                 I915_WRITE(DSPFW7_CHV,
1000                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1001                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1002                 I915_WRITE(DSPFW8_CHV,
1003                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1004                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1005                 I915_WRITE(DSPFW9_CHV,
1006                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1007                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1008                 I915_WRITE(DSPHOWM,
1009                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1010                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1011                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1012                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1013                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1014                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1015                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1016                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1017                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1018                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1019         } else {
1020                 I915_WRITE(DSPFW7,
1021                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1022                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1023                 I915_WRITE(DSPHOWM,
1024                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1025                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1026                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1027                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1028                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1029                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1030                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1031         }
1032
1033         POSTING_READ(DSPFW1);
1034 }
1035
1036 #undef FW_WM_VLV
1037
1038 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1039 {
1040         /* all latencies in usec */
1041         dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1042         dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1043         dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1044
1045         dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1046 }
1047
1048 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1049 {
1050         /*
1051          * DSPCNTR[13] supposedly controls whether the
1052          * primary plane can use the FIFO space otherwise
1053          * reserved for the sprite plane. It's not 100% clear
1054          * what the actual FIFO size is, but it looks like we
1055          * can happily set both primary and sprite watermarks
1056          * up to 127 cachelines. So that would seem to mean
1057          * that either DSPCNTR[13] doesn't do anything, or that
1058          * the total FIFO is >= 256 cachelines in size. Either
1059          * way, we don't seem to have to worry about this
1060          * repartitioning as the maximum watermark value the
1061          * register can hold for each plane is lower than the
1062          * minimum FIFO size.
1063          */
1064         switch (plane_id) {
1065         case PLANE_CURSOR:
1066                 return 63;
1067         case PLANE_PRIMARY:
1068                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1069         case PLANE_SPRITE0:
1070                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1071         default:
1072                 MISSING_CASE(plane_id);
1073                 return 0;
1074         }
1075 }
1076
1077 static int g4x_fbc_fifo_size(int level)
1078 {
1079         switch (level) {
1080         case G4X_WM_LEVEL_SR:
1081                 return 7;
1082         case G4X_WM_LEVEL_HPLL:
1083                 return 15;
1084         default:
1085                 MISSING_CASE(level);
1086                 return 0;
1087         }
1088 }
1089
1090 static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1091                                const struct intel_plane_state *plane_state,
1092                                int level)
1093 {
1094         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1095         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1096         const struct drm_display_mode *adjusted_mode =
1097                 &crtc_state->base.adjusted_mode;
1098         unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1099         unsigned int clock, htotal, cpp, width, wm;
1100
1101         if (latency == 0)
1102                 return USHRT_MAX;
1103
1104         if (!intel_wm_plane_visible(crtc_state, plane_state))
1105                 return 0;
1106
1107         /*
1108          * Not 100% sure which way ELK should go here as the
1109          * spec only says CL/CTG should assume 32bpp and BW
1110          * doesn't need to. But as these things followed the
1111          * mobile vs. desktop lines on gen3 as well, let's
1112          * assume ELK doesn't need this.
1113          *
1114          * The spec also fails to list such a restriction for
1115          * the HPLL watermark, which seems a little strange.
1116          * Let's use 32bpp for the HPLL watermark as well.
1117          */
1118         if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1119             level != G4X_WM_LEVEL_NORMAL)
1120                 cpp = 4;
1121         else
1122                 cpp = plane_state->base.fb->format->cpp[0];
1123
1124         clock = adjusted_mode->crtc_clock;
1125         htotal = adjusted_mode->crtc_htotal;
1126
1127         if (plane->id == PLANE_CURSOR)
1128                 width = plane_state->base.crtc_w;
1129         else
1130                 width = drm_rect_width(&plane_state->base.dst);
1131
1132         if (plane->id == PLANE_CURSOR) {
1133                 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1134         } else if (plane->id == PLANE_PRIMARY &&
1135                    level == G4X_WM_LEVEL_NORMAL) {
1136                 wm = intel_wm_method1(clock, cpp, latency);
1137         } else {
1138                 unsigned int small, large;
1139
1140                 small = intel_wm_method1(clock, cpp, latency);
1141                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1142
1143                 wm = min(small, large);
1144         }
1145
1146         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1147                               width, cpp);
1148
1149         wm = DIV_ROUND_UP(wm, 64) + 2;
1150
1151         return min_t(unsigned int, wm, USHRT_MAX);
1152 }
1153
1154 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1155                                  int level, enum plane_id plane_id, u16 value)
1156 {
1157         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1158         bool dirty = false;
1159
1160         for (; level < intel_wm_num_levels(dev_priv); level++) {
1161                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1162
1163                 dirty |= raw->plane[plane_id] != value;
1164                 raw->plane[plane_id] = value;
1165         }
1166
1167         return dirty;
1168 }
1169
1170 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1171                                int level, u16 value)
1172 {
1173         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1174         bool dirty = false;
1175
1176         /* NORMAL level doesn't have an FBC watermark */
1177         level = max(level, G4X_WM_LEVEL_SR);
1178
1179         for (; level < intel_wm_num_levels(dev_priv); level++) {
1180                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1181
1182                 dirty |= raw->fbc != value;
1183                 raw->fbc = value;
1184         }
1185
1186         return dirty;
1187 }
1188
1189 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
1190                                    const struct intel_plane_state *pstate,
1191                                    uint32_t pri_val);
1192
1193 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1194                                      const struct intel_plane_state *plane_state)
1195 {
1196         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1197         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1198         enum plane_id plane_id = plane->id;
1199         bool dirty = false;
1200         int level;
1201
1202         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1203                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1204                 if (plane_id == PLANE_PRIMARY)
1205                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1206                 goto out;
1207         }
1208
1209         for (level = 0; level < num_levels; level++) {
1210                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1211                 int wm, max_wm;
1212
1213                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1214                 max_wm = g4x_plane_fifo_size(plane_id, level);
1215
1216                 if (wm > max_wm)
1217                         break;
1218
1219                 dirty |= raw->plane[plane_id] != wm;
1220                 raw->plane[plane_id] = wm;
1221
1222                 if (plane_id != PLANE_PRIMARY ||
1223                     level == G4X_WM_LEVEL_NORMAL)
1224                         continue;
1225
1226                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1227                                         raw->plane[plane_id]);
1228                 max_wm = g4x_fbc_fifo_size(level);
1229
1230                 /*
1231                  * FBC wm is not mandatory as we
1232                  * can always just disable its use.
1233                  */
1234                 if (wm > max_wm)
1235                         wm = USHRT_MAX;
1236
1237                 dirty |= raw->fbc != wm;
1238                 raw->fbc = wm;
1239         }
1240
1241         /* mark watermarks as invalid */
1242         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1243
1244         if (plane_id == PLANE_PRIMARY)
1245                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1246
1247  out:
1248         if (dirty) {
1249                 DRM_DEBUG_KMS("%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1250                               plane->base.name,
1251                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1252                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1253                               crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1254
1255                 if (plane_id == PLANE_PRIMARY)
1256                         DRM_DEBUG_KMS("FBC watermarks: SR=%d, HPLL=%d\n",
1257                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1258                                       crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1259         }
1260
1261         return dirty;
1262 }
1263
1264 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1265                                       enum plane_id plane_id, int level)
1266 {
1267         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1268
1269         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1270 }
1271
1272 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1273                                      int level)
1274 {
1275         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1276
1277         if (level > dev_priv->wm.max_level)
1278                 return false;
1279
1280         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1281                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1282                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1283 }
1284
1285 /* mark all levels starting from 'level' as invalid */
1286 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1287                                struct g4x_wm_state *wm_state, int level)
1288 {
1289         if (level <= G4X_WM_LEVEL_NORMAL) {
1290                 enum plane_id plane_id;
1291
1292                 for_each_plane_id_on_crtc(crtc, plane_id)
1293                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1294         }
1295
1296         if (level <= G4X_WM_LEVEL_SR) {
1297                 wm_state->cxsr = false;
1298                 wm_state->sr.cursor = USHRT_MAX;
1299                 wm_state->sr.plane = USHRT_MAX;
1300                 wm_state->sr.fbc = USHRT_MAX;
1301         }
1302
1303         if (level <= G4X_WM_LEVEL_HPLL) {
1304                 wm_state->hpll_en = false;
1305                 wm_state->hpll.cursor = USHRT_MAX;
1306                 wm_state->hpll.plane = USHRT_MAX;
1307                 wm_state->hpll.fbc = USHRT_MAX;
1308         }
1309 }
1310
1311 static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1312 {
1313         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1314         struct intel_atomic_state *state =
1315                 to_intel_atomic_state(crtc_state->base.state);
1316         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1317         int num_active_planes = hweight32(crtc_state->active_planes &
1318                                           ~BIT(PLANE_CURSOR));
1319         const struct g4x_pipe_wm *raw;
1320         const struct intel_plane_state *old_plane_state;
1321         const struct intel_plane_state *new_plane_state;
1322         struct intel_plane *plane;
1323         enum plane_id plane_id;
1324         int i, level;
1325         unsigned int dirty = 0;
1326
1327         for_each_oldnew_intel_plane_in_state(state, plane,
1328                                              old_plane_state,
1329                                              new_plane_state, i) {
1330                 if (new_plane_state->base.crtc != &crtc->base &&
1331                     old_plane_state->base.crtc != &crtc->base)
1332                         continue;
1333
1334                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1335                         dirty |= BIT(plane->id);
1336         }
1337
1338         if (!dirty)
1339                 return 0;
1340
1341         level = G4X_WM_LEVEL_NORMAL;
1342         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1343                 goto out;
1344
1345         raw = &crtc_state->wm.g4x.raw[level];
1346         for_each_plane_id_on_crtc(crtc, plane_id)
1347                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1348
1349         level = G4X_WM_LEVEL_SR;
1350
1351         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1352                 goto out;
1353
1354         raw = &crtc_state->wm.g4x.raw[level];
1355         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1356         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1357         wm_state->sr.fbc = raw->fbc;
1358
1359         wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1360
1361         level = G4X_WM_LEVEL_HPLL;
1362
1363         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1364                 goto out;
1365
1366         raw = &crtc_state->wm.g4x.raw[level];
1367         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1368         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1369         wm_state->hpll.fbc = raw->fbc;
1370
1371         wm_state->hpll_en = wm_state->cxsr;
1372
1373         level++;
1374
1375  out:
1376         if (level == G4X_WM_LEVEL_NORMAL)
1377                 return -EINVAL;
1378
1379         /* invalidate the higher levels */
1380         g4x_invalidate_wms(crtc, wm_state, level);
1381
1382         /*
1383          * Determine if the FBC watermark(s) can be used. IF
1384          * this isn't the case we prefer to disable the FBC
1385          ( watermark(s) rather than disable the SR/HPLL
1386          * level(s) entirely.
1387          */
1388         wm_state->fbc_en = level > G4X_WM_LEVEL_NORMAL;
1389
1390         if (level >= G4X_WM_LEVEL_SR &&
1391             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1392                 wm_state->fbc_en = false;
1393         else if (level >= G4X_WM_LEVEL_HPLL &&
1394                  wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1395                 wm_state->fbc_en = false;
1396
1397         return 0;
1398 }
1399
1400 static int g4x_compute_intermediate_wm(struct drm_device *dev,
1401                                        struct intel_crtc *crtc,
1402                                        struct intel_crtc_state *new_crtc_state)
1403 {
1404         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1405         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1406         struct intel_atomic_state *intel_state =
1407                 to_intel_atomic_state(new_crtc_state->base.state);
1408         const struct intel_crtc_state *old_crtc_state =
1409                 intel_atomic_get_old_crtc_state(intel_state, crtc);
1410         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1411         enum plane_id plane_id;
1412
1413         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
1414                 *intermediate = *optimal;
1415
1416                 intermediate->cxsr = false;
1417                 intermediate->hpll_en = false;
1418                 goto out;
1419         }
1420
1421         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1422                 !new_crtc_state->disable_cxsr;
1423         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1424                 !new_crtc_state->disable_cxsr;
1425         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1426
1427         for_each_plane_id_on_crtc(crtc, plane_id) {
1428                 intermediate->wm.plane[plane_id] =
1429                         max(optimal->wm.plane[plane_id],
1430                             active->wm.plane[plane_id]);
1431
1432                 WARN_ON(intermediate->wm.plane[plane_id] >
1433                         g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1434         }
1435
1436         intermediate->sr.plane = max(optimal->sr.plane,
1437                                      active->sr.plane);
1438         intermediate->sr.cursor = max(optimal->sr.cursor,
1439                                       active->sr.cursor);
1440         intermediate->sr.fbc = max(optimal->sr.fbc,
1441                                    active->sr.fbc);
1442
1443         intermediate->hpll.plane = max(optimal->hpll.plane,
1444                                        active->hpll.plane);
1445         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1446                                         active->hpll.cursor);
1447         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1448                                      active->hpll.fbc);
1449
1450         WARN_ON((intermediate->sr.plane >
1451                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1452                  intermediate->sr.cursor >
1453                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1454                 intermediate->cxsr);
1455         WARN_ON((intermediate->sr.plane >
1456                  g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1457                  intermediate->sr.cursor >
1458                  g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1459                 intermediate->hpll_en);
1460
1461         WARN_ON(intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1462                 intermediate->fbc_en && intermediate->cxsr);
1463         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1464                 intermediate->fbc_en && intermediate->hpll_en);
1465
1466 out:
1467         /*
1468          * If our intermediate WM are identical to the final WM, then we can
1469          * omit the post-vblank programming; only update if it's different.
1470          */
1471         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1472                 new_crtc_state->wm.need_postvbl_update = true;
1473
1474         return 0;
1475 }
1476
1477 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1478                          struct g4x_wm_values *wm)
1479 {
1480         struct intel_crtc *crtc;
1481         int num_active_crtcs = 0;
1482
1483         wm->cxsr = true;
1484         wm->hpll_en = true;
1485         wm->fbc_en = true;
1486
1487         for_each_intel_crtc(&dev_priv->drm, crtc) {
1488                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1489
1490                 if (!crtc->active)
1491                         continue;
1492
1493                 if (!wm_state->cxsr)
1494                         wm->cxsr = false;
1495                 if (!wm_state->hpll_en)
1496                         wm->hpll_en = false;
1497                 if (!wm_state->fbc_en)
1498                         wm->fbc_en = false;
1499
1500                 num_active_crtcs++;
1501         }
1502
1503         if (num_active_crtcs != 1) {
1504                 wm->cxsr = false;
1505                 wm->hpll_en = false;
1506                 wm->fbc_en = false;
1507         }
1508
1509         for_each_intel_crtc(&dev_priv->drm, crtc) {
1510                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1511                 enum pipe pipe = crtc->pipe;
1512
1513                 wm->pipe[pipe] = wm_state->wm;
1514                 if (crtc->active && wm->cxsr)
1515                         wm->sr = wm_state->sr;
1516                 if (crtc->active && wm->hpll_en)
1517                         wm->hpll = wm_state->hpll;
1518         }
1519 }
1520
1521 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1522 {
1523         struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1524         struct g4x_wm_values new_wm = {};
1525
1526         g4x_merge_wm(dev_priv, &new_wm);
1527
1528         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1529                 return;
1530
1531         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1532                 _intel_set_memory_cxsr(dev_priv, false);
1533
1534         g4x_write_wm_values(dev_priv, &new_wm);
1535
1536         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1537                 _intel_set_memory_cxsr(dev_priv, true);
1538
1539         *old_wm = new_wm;
1540 }
1541
1542 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1543                                    struct intel_crtc_state *crtc_state)
1544 {
1545         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1546         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1547
1548         mutex_lock(&dev_priv->wm.wm_mutex);
1549         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1550         g4x_program_watermarks(dev_priv);
1551         mutex_unlock(&dev_priv->wm.wm_mutex);
1552 }
1553
1554 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1555                                     struct intel_crtc_state *crtc_state)
1556 {
1557         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1558         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
1559
1560         if (!crtc_state->wm.need_postvbl_update)
1561                 return;
1562
1563         mutex_lock(&dev_priv->wm.wm_mutex);
1564         intel_crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1565         g4x_program_watermarks(dev_priv);
1566         mutex_unlock(&dev_priv->wm.wm_mutex);
1567 }
1568
1569 /* latency must be in 0.1us units. */
1570 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1571                                    unsigned int htotal,
1572                                    unsigned int width,
1573                                    unsigned int cpp,
1574                                    unsigned int latency)
1575 {
1576         unsigned int ret;
1577
1578         ret = intel_wm_method2(pixel_rate, htotal,
1579                                width, cpp, latency);
1580         ret = DIV_ROUND_UP(ret, 64);
1581
1582         return ret;
1583 }
1584
1585 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1586 {
1587         /* all latencies in usec */
1588         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1589
1590         dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1591
1592         if (IS_CHERRYVIEW(dev_priv)) {
1593                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1594                 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1595
1596                 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1597         }
1598 }
1599
1600 static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1601                                      const struct intel_plane_state *plane_state,
1602                                      int level)
1603 {
1604         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1605         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1606         const struct drm_display_mode *adjusted_mode =
1607                 &crtc_state->base.adjusted_mode;
1608         unsigned int clock, htotal, cpp, width, wm;
1609
1610         if (dev_priv->wm.pri_latency[level] == 0)
1611                 return USHRT_MAX;
1612
1613         if (!intel_wm_plane_visible(crtc_state, plane_state))
1614                 return 0;
1615
1616         cpp = plane_state->base.fb->format->cpp[0];
1617         clock = adjusted_mode->crtc_clock;
1618         htotal = adjusted_mode->crtc_htotal;
1619         width = crtc_state->pipe_src_w;
1620
1621         if (plane->id == PLANE_CURSOR) {
1622                 /*
1623                  * FIXME the formula gives values that are
1624                  * too big for the cursor FIFO, and hence we
1625                  * would never be able to use cursors. For
1626                  * now just hardcode the watermark.
1627                  */
1628                 wm = 63;
1629         } else {
1630                 wm = vlv_wm_method2(clock, htotal, width, cpp,
1631                                     dev_priv->wm.pri_latency[level] * 10);
1632         }
1633
1634         return min_t(unsigned int, wm, USHRT_MAX);
1635 }
1636
1637 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1638 {
1639         return (active_planes & (BIT(PLANE_SPRITE0) |
1640                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1641 }
1642
1643 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1644 {
1645         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1646         const struct g4x_pipe_wm *raw =
1647                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1648         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1649         unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1650         int num_active_planes = hweight32(active_planes);
1651         const int fifo_size = 511;
1652         int fifo_extra, fifo_left = fifo_size;
1653         int sprite0_fifo_extra = 0;
1654         unsigned int total_rate;
1655         enum plane_id plane_id;
1656
1657         /*
1658          * When enabling sprite0 after sprite1 has already been enabled
1659          * we tend to get an underrun unless sprite0 already has some
1660          * FIFO space allcoated. Hence we always allocate at least one
1661          * cacheline for sprite0 whenever sprite1 is enabled.
1662          *
1663          * All other plane enable sequences appear immune to this problem.
1664          */
1665         if (vlv_need_sprite0_fifo_workaround(active_planes))
1666                 sprite0_fifo_extra = 1;
1667
1668         total_rate = raw->plane[PLANE_PRIMARY] +
1669                 raw->plane[PLANE_SPRITE0] +
1670                 raw->plane[PLANE_SPRITE1] +
1671                 sprite0_fifo_extra;
1672
1673         if (total_rate > fifo_size)
1674                 return -EINVAL;
1675
1676         if (total_rate == 0)
1677                 total_rate = 1;
1678
1679         for_each_plane_id_on_crtc(crtc, plane_id) {
1680                 unsigned int rate;
1681
1682                 if ((active_planes & BIT(plane_id)) == 0) {
1683                         fifo_state->plane[plane_id] = 0;
1684                         continue;
1685                 }
1686
1687                 rate = raw->plane[plane_id];
1688                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1689                 fifo_left -= fifo_state->plane[plane_id];
1690         }
1691
1692         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1693         fifo_left -= sprite0_fifo_extra;
1694
1695         fifo_state->plane[PLANE_CURSOR] = 63;
1696
1697         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1698
1699         /* spread the remainder evenly */
1700         for_each_plane_id_on_crtc(crtc, plane_id) {
1701                 int plane_extra;
1702
1703                 if (fifo_left == 0)
1704                         break;
1705
1706                 if ((active_planes & BIT(plane_id)) == 0)
1707                         continue;
1708
1709                 plane_extra = min(fifo_extra, fifo_left);
1710                 fifo_state->plane[plane_id] += plane_extra;
1711                 fifo_left -= plane_extra;
1712         }
1713
1714         WARN_ON(active_planes != 0 && fifo_left != 0);
1715
1716         /* give it all to the first plane if none are active */
1717         if (active_planes == 0) {
1718                 WARN_ON(fifo_left != fifo_size);
1719                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1720         }
1721
1722         return 0;
1723 }
1724
1725 /* mark all levels starting from 'level' as invalid */
1726 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1727                                struct vlv_wm_state *wm_state, int level)
1728 {
1729         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1730
1731         for (; level < intel_wm_num_levels(dev_priv); level++) {
1732                 enum plane_id plane_id;
1733
1734                 for_each_plane_id_on_crtc(crtc, plane_id)
1735                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1736
1737                 wm_state->sr[level].cursor = USHRT_MAX;
1738                 wm_state->sr[level].plane = USHRT_MAX;
1739         }
1740 }
1741
1742 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1743 {
1744         if (wm > fifo_size)
1745                 return USHRT_MAX;
1746         else
1747                 return fifo_size - wm;
1748 }
1749
1750 /*
1751  * Starting from 'level' set all higher
1752  * levels to 'value' in the "raw" watermarks.
1753  */
1754 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1755                                  int level, enum plane_id plane_id, u16 value)
1756 {
1757         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
1758         int num_levels = intel_wm_num_levels(dev_priv);
1759         bool dirty = false;
1760
1761         for (; level < num_levels; level++) {
1762                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1763
1764                 dirty |= raw->plane[plane_id] != value;
1765                 raw->plane[plane_id] = value;
1766         }
1767
1768         return dirty;
1769 }
1770
1771 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1772                                      const struct intel_plane_state *plane_state)
1773 {
1774         struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
1775         enum plane_id plane_id = plane->id;
1776         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1777         int level;
1778         bool dirty = false;
1779
1780         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1781                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1782                 goto out;
1783         }
1784
1785         for (level = 0; level < num_levels; level++) {
1786                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1787                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1788                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1789
1790                 if (wm > max_wm)
1791                         break;
1792
1793                 dirty |= raw->plane[plane_id] != wm;
1794                 raw->plane[plane_id] = wm;
1795         }
1796
1797         /* mark all higher levels as invalid */
1798         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1799
1800 out:
1801         if (dirty)
1802                 DRM_DEBUG_KMS("%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1803                               plane->base.name,
1804                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1805                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1806                               crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1807
1808         return dirty;
1809 }
1810
1811 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1812                                       enum plane_id plane_id, int level)
1813 {
1814         const struct g4x_pipe_wm *raw =
1815                 &crtc_state->wm.vlv.raw[level];
1816         const struct vlv_fifo_state *fifo_state =
1817                 &crtc_state->wm.vlv.fifo_state;
1818
1819         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1820 }
1821
1822 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1823 {
1824         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1825                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1826                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1827                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1828 }
1829
1830 static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1831 {
1832         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1833         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1834         struct intel_atomic_state *state =
1835                 to_intel_atomic_state(crtc_state->base.state);
1836         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1837         const struct vlv_fifo_state *fifo_state =
1838                 &crtc_state->wm.vlv.fifo_state;
1839         int num_active_planes = hweight32(crtc_state->active_planes &
1840                                           ~BIT(PLANE_CURSOR));
1841         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->base);
1842         const struct intel_plane_state *old_plane_state;
1843         const struct intel_plane_state *new_plane_state;
1844         struct intel_plane *plane;
1845         enum plane_id plane_id;
1846         int level, ret, i;
1847         unsigned int dirty = 0;
1848
1849         for_each_oldnew_intel_plane_in_state(state, plane,
1850                                              old_plane_state,
1851                                              new_plane_state, i) {
1852                 if (new_plane_state->base.crtc != &crtc->base &&
1853                     old_plane_state->base.crtc != &crtc->base)
1854                         continue;
1855
1856                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1857                         dirty |= BIT(plane->id);
1858         }
1859
1860         /*
1861          * DSPARB registers may have been reset due to the
1862          * power well being turned off. Make sure we restore
1863          * them to a consistent state even if no primary/sprite
1864          * planes are initially active.
1865          */
1866         if (needs_modeset)
1867                 crtc_state->fifo_changed = true;
1868
1869         if (!dirty)
1870                 return 0;
1871
1872         /* cursor changes don't warrant a FIFO recompute */
1873         if (dirty & ~BIT(PLANE_CURSOR)) {
1874                 const struct intel_crtc_state *old_crtc_state =
1875                         intel_atomic_get_old_crtc_state(state, crtc);
1876                 const struct vlv_fifo_state *old_fifo_state =
1877                         &old_crtc_state->wm.vlv.fifo_state;
1878
1879                 ret = vlv_compute_fifo(crtc_state);
1880                 if (ret)
1881                         return ret;
1882
1883                 if (needs_modeset ||
1884                     memcmp(old_fifo_state, fifo_state,
1885                            sizeof(*fifo_state)) != 0)
1886                         crtc_state->fifo_changed = true;
1887         }
1888
1889         /* initially allow all levels */
1890         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1891         /*
1892          * Note that enabling cxsr with no primary/sprite planes
1893          * enabled can wedge the pipe. Hence we only allow cxsr
1894          * with exactly one enabled primary/sprite plane.
1895          */
1896         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1897
1898         for (level = 0; level < wm_state->num_levels; level++) {
1899                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1900                 const int sr_fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1901
1902                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1903                         break;
1904
1905                 for_each_plane_id_on_crtc(crtc, plane_id) {
1906                         wm_state->wm[level].plane[plane_id] =
1907                                 vlv_invert_wm_value(raw->plane[plane_id],
1908                                                     fifo_state->plane[plane_id]);
1909                 }
1910
1911                 wm_state->sr[level].plane =
1912                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1913                                                  raw->plane[PLANE_SPRITE0],
1914                                                  raw->plane[PLANE_SPRITE1]),
1915                                             sr_fifo_size);
1916
1917                 wm_state->sr[level].cursor =
1918                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1919                                             63);
1920         }
1921
1922         if (level == 0)
1923                 return -EINVAL;
1924
1925         /* limit to only levels we can actually handle */
1926         wm_state->num_levels = level;
1927
1928         /* invalidate the higher levels */
1929         vlv_invalidate_wms(crtc, wm_state, level);
1930
1931         return 0;
1932 }
1933
1934 #define VLV_FIFO(plane, value) \
1935         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1936
1937 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1938                                    struct intel_crtc_state *crtc_state)
1939 {
1940         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
1941         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1942         const struct vlv_fifo_state *fifo_state =
1943                 &crtc_state->wm.vlv.fifo_state;
1944         int sprite0_start, sprite1_start, fifo_size;
1945
1946         if (!crtc_state->fifo_changed)
1947                 return;
1948
1949         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1950         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1951         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1952
1953         WARN_ON(fifo_state->plane[PLANE_CURSOR] != 63);
1954         WARN_ON(fifo_size != 511);
1955
1956         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1957
1958         /*
1959          * uncore.lock serves a double purpose here. It allows us to
1960          * use the less expensive I915_{READ,WRITE}_FW() functions, and
1961          * it protects the DSPARB registers from getting clobbered by
1962          * parallel updates from multiple pipes.
1963          *
1964          * intel_pipe_update_start() has already disabled interrupts
1965          * for us, so a plain spin_lock() is sufficient here.
1966          */
1967         spin_lock(&dev_priv->uncore.lock);
1968
1969         switch (crtc->pipe) {
1970                 uint32_t dsparb, dsparb2, dsparb3;
1971         case PIPE_A:
1972                 dsparb = I915_READ_FW(DSPARB);
1973                 dsparb2 = I915_READ_FW(DSPARB2);
1974
1975                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1976                             VLV_FIFO(SPRITEB, 0xff));
1977                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1978                            VLV_FIFO(SPRITEB, sprite1_start));
1979
1980                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1981                              VLV_FIFO(SPRITEB_HI, 0x1));
1982                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1983                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1984
1985                 I915_WRITE_FW(DSPARB, dsparb);
1986                 I915_WRITE_FW(DSPARB2, dsparb2);
1987                 break;
1988         case PIPE_B:
1989                 dsparb = I915_READ_FW(DSPARB);
1990                 dsparb2 = I915_READ_FW(DSPARB2);
1991
1992                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1993                             VLV_FIFO(SPRITED, 0xff));
1994                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1995                            VLV_FIFO(SPRITED, sprite1_start));
1996
1997                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1998                              VLV_FIFO(SPRITED_HI, 0xff));
1999                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2000                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2001
2002                 I915_WRITE_FW(DSPARB, dsparb);
2003                 I915_WRITE_FW(DSPARB2, dsparb2);
2004                 break;
2005         case PIPE_C:
2006                 dsparb3 = I915_READ_FW(DSPARB3);
2007                 dsparb2 = I915_READ_FW(DSPARB2);
2008
2009                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2010                              VLV_FIFO(SPRITEF, 0xff));
2011                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2012                             VLV_FIFO(SPRITEF, sprite1_start));
2013
2014                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2015                              VLV_FIFO(SPRITEF_HI, 0xff));
2016                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2017                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2018
2019                 I915_WRITE_FW(DSPARB3, dsparb3);
2020                 I915_WRITE_FW(DSPARB2, dsparb2);
2021                 break;
2022         default:
2023                 break;
2024         }
2025
2026         POSTING_READ_FW(DSPARB);
2027
2028         spin_unlock(&dev_priv->uncore.lock);
2029 }
2030
2031 #undef VLV_FIFO
2032
2033 static int vlv_compute_intermediate_wm(struct drm_device *dev,
2034                                        struct intel_crtc *crtc,
2035                                        struct intel_crtc_state *new_crtc_state)
2036 {
2037         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2038         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2039         struct intel_atomic_state *intel_state =
2040                 to_intel_atomic_state(new_crtc_state->base.state);
2041         const struct intel_crtc_state *old_crtc_state =
2042                 intel_atomic_get_old_crtc_state(intel_state, crtc);
2043         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2044         int level;
2045
2046         if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
2047                 *intermediate = *optimal;
2048
2049                 intermediate->cxsr = false;
2050                 goto out;
2051         }
2052
2053         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2054         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2055                 !new_crtc_state->disable_cxsr;
2056
2057         for (level = 0; level < intermediate->num_levels; level++) {
2058                 enum plane_id plane_id;
2059
2060                 for_each_plane_id_on_crtc(crtc, plane_id) {
2061                         intermediate->wm[level].plane[plane_id] =
2062                                 min(optimal->wm[level].plane[plane_id],
2063                                     active->wm[level].plane[plane_id]);
2064                 }
2065
2066                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2067                                                     active->sr[level].plane);
2068                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2069                                                      active->sr[level].cursor);
2070         }
2071
2072         vlv_invalidate_wms(crtc, intermediate, level);
2073
2074 out:
2075         /*
2076          * If our intermediate WM are identical to the final WM, then we can
2077          * omit the post-vblank programming; only update if it's different.
2078          */
2079         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2080                 new_crtc_state->wm.need_postvbl_update = true;
2081
2082         return 0;
2083 }
2084
2085 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2086                          struct vlv_wm_values *wm)
2087 {
2088         struct intel_crtc *crtc;
2089         int num_active_crtcs = 0;
2090
2091         wm->level = dev_priv->wm.max_level;
2092         wm->cxsr = true;
2093
2094         for_each_intel_crtc(&dev_priv->drm, crtc) {
2095                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2096
2097                 if (!crtc->active)
2098                         continue;
2099
2100                 if (!wm_state->cxsr)
2101                         wm->cxsr = false;
2102
2103                 num_active_crtcs++;
2104                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2105         }
2106
2107         if (num_active_crtcs != 1)
2108                 wm->cxsr = false;
2109
2110         if (num_active_crtcs > 1)
2111                 wm->level = VLV_WM_LEVEL_PM2;
2112
2113         for_each_intel_crtc(&dev_priv->drm, crtc) {
2114                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2115                 enum pipe pipe = crtc->pipe;
2116
2117                 wm->pipe[pipe] = wm_state->wm[wm->level];
2118                 if (crtc->active && wm->cxsr)
2119                         wm->sr = wm_state->sr[wm->level];
2120
2121                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2122                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2123                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2124                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2125         }
2126 }
2127
2128 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2129 {
2130         struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2131         struct vlv_wm_values new_wm = {};
2132
2133         vlv_merge_wm(dev_priv, &new_wm);
2134
2135         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2136                 return;
2137
2138         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2139                 chv_set_memory_dvfs(dev_priv, false);
2140
2141         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2142                 chv_set_memory_pm5(dev_priv, false);
2143
2144         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2145                 _intel_set_memory_cxsr(dev_priv, false);
2146
2147         vlv_write_wm_values(dev_priv, &new_wm);
2148
2149         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2150                 _intel_set_memory_cxsr(dev_priv, true);
2151
2152         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2153                 chv_set_memory_pm5(dev_priv, true);
2154
2155         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2156                 chv_set_memory_dvfs(dev_priv, true);
2157
2158         *old_wm = new_wm;
2159 }
2160
2161 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2162                                    struct intel_crtc_state *crtc_state)
2163 {
2164         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2165         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
2166
2167         mutex_lock(&dev_priv->wm.wm_mutex);
2168         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2169         vlv_program_watermarks(dev_priv);
2170         mutex_unlock(&dev_priv->wm.wm_mutex);
2171 }
2172
2173 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2174                                     struct intel_crtc_state *crtc_state)
2175 {
2176         struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
2177         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2178
2179         if (!crtc_state->wm.need_postvbl_update)
2180                 return;
2181
2182         mutex_lock(&dev_priv->wm.wm_mutex);
2183         intel_crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2184         vlv_program_watermarks(dev_priv);
2185         mutex_unlock(&dev_priv->wm.wm_mutex);
2186 }
2187
2188 static void i965_update_wm(struct intel_crtc *unused_crtc)
2189 {
2190         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2191         struct intel_crtc *crtc;
2192         int srwm = 1;
2193         int cursor_sr = 16;
2194         bool cxsr_enabled;
2195
2196         /* Calc sr entries for one plane configs */
2197         crtc = single_enabled_crtc(dev_priv);
2198         if (crtc) {
2199                 /* self-refresh has much higher latency */
2200                 static const int sr_latency_ns = 12000;
2201                 const struct drm_display_mode *adjusted_mode =
2202                         &crtc->config->base.adjusted_mode;
2203                 const struct drm_framebuffer *fb =
2204                         crtc->base.primary->state->fb;
2205                 int clock = adjusted_mode->crtc_clock;
2206                 int htotal = adjusted_mode->crtc_htotal;
2207                 int hdisplay = crtc->config->pipe_src_w;
2208                 int cpp = fb->format->cpp[0];
2209                 int entries;
2210
2211                 entries = intel_wm_method2(clock, htotal,
2212                                            hdisplay, cpp, sr_latency_ns / 100);
2213                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2214                 srwm = I965_FIFO_SIZE - entries;
2215                 if (srwm < 0)
2216                         srwm = 1;
2217                 srwm &= 0x1ff;
2218                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
2219                               entries, srwm);
2220
2221                 entries = intel_wm_method2(clock, htotal,
2222                                            crtc->base.cursor->state->crtc_w, 4,
2223                                            sr_latency_ns / 100);
2224                 entries = DIV_ROUND_UP(entries,
2225                                        i965_cursor_wm_info.cacheline_size) +
2226                         i965_cursor_wm_info.guard_size;
2227
2228                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2229                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2230                         cursor_sr = i965_cursor_wm_info.max_wm;
2231
2232                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
2233                               "cursor %d\n", srwm, cursor_sr);
2234
2235                 cxsr_enabled = true;
2236         } else {
2237                 cxsr_enabled = false;
2238                 /* Turn off self refresh if both pipes are enabled */
2239                 intel_set_memory_cxsr(dev_priv, false);
2240         }
2241
2242         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2243                       srwm);
2244
2245         /* 965 has limitations... */
2246         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
2247                    FW_WM(8, CURSORB) |
2248                    FW_WM(8, PLANEB) |
2249                    FW_WM(8, PLANEA));
2250         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
2251                    FW_WM(8, PLANEC_OLD));
2252         /* update cursor SR watermark */
2253         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2254
2255         if (cxsr_enabled)
2256                 intel_set_memory_cxsr(dev_priv, true);
2257 }
2258
2259 #undef FW_WM
2260
2261 static void i9xx_update_wm(struct intel_crtc *unused_crtc)
2262 {
2263         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2264         const struct intel_watermark_params *wm_info;
2265         uint32_t fwater_lo;
2266         uint32_t fwater_hi;
2267         int cwm, srwm = 1;
2268         int fifo_size;
2269         int planea_wm, planeb_wm;
2270         struct intel_crtc *crtc, *enabled = NULL;
2271
2272         if (IS_I945GM(dev_priv))
2273                 wm_info = &i945_wm_info;
2274         else if (!IS_GEN2(dev_priv))
2275                 wm_info = &i915_wm_info;
2276         else
2277                 wm_info = &i830_a_wm_info;
2278
2279         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2280         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
2281         if (intel_crtc_active(crtc)) {
2282                 const struct drm_display_mode *adjusted_mode =
2283                         &crtc->config->base.adjusted_mode;
2284                 const struct drm_framebuffer *fb =
2285                         crtc->base.primary->state->fb;
2286                 int cpp;
2287
2288                 if (IS_GEN2(dev_priv))
2289                         cpp = 4;
2290                 else
2291                         cpp = fb->format->cpp[0];
2292
2293                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2294                                                wm_info, fifo_size, cpp,
2295                                                pessimal_latency_ns);
2296                 enabled = crtc;
2297         } else {
2298                 planea_wm = fifo_size - wm_info->guard_size;
2299                 if (planea_wm > (long)wm_info->max_wm)
2300                         planea_wm = wm_info->max_wm;
2301         }
2302
2303         if (IS_GEN2(dev_priv))
2304                 wm_info = &i830_bc_wm_info;
2305
2306         fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2307         crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
2308         if (intel_crtc_active(crtc)) {
2309                 const struct drm_display_mode *adjusted_mode =
2310                         &crtc->config->base.adjusted_mode;
2311                 const struct drm_framebuffer *fb =
2312                         crtc->base.primary->state->fb;
2313                 int cpp;
2314
2315                 if (IS_GEN2(dev_priv))
2316                         cpp = 4;
2317                 else
2318                         cpp = fb->format->cpp[0];
2319
2320                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2321                                                wm_info, fifo_size, cpp,
2322                                                pessimal_latency_ns);
2323                 if (enabled == NULL)
2324                         enabled = crtc;
2325                 else
2326                         enabled = NULL;
2327         } else {
2328                 planeb_wm = fifo_size - wm_info->guard_size;
2329                 if (planeb_wm > (long)wm_info->max_wm)
2330                         planeb_wm = wm_info->max_wm;
2331         }
2332
2333         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2334
2335         if (IS_I915GM(dev_priv) && enabled) {
2336                 struct drm_i915_gem_object *obj;
2337
2338                 obj = intel_fb_obj(enabled->base.primary->state->fb);
2339
2340                 /* self-refresh seems busted with untiled */
2341                 if (!i915_gem_object_is_tiled(obj))
2342                         enabled = NULL;
2343         }
2344
2345         /*
2346          * Overlay gets an aggressive default since video jitter is bad.
2347          */
2348         cwm = 2;
2349
2350         /* Play safe and disable self-refresh before adjusting watermarks. */
2351         intel_set_memory_cxsr(dev_priv, false);
2352
2353         /* Calc sr entries for one plane configs */
2354         if (HAS_FW_BLC(dev_priv) && enabled) {
2355                 /* self-refresh has much higher latency */
2356                 static const int sr_latency_ns = 6000;
2357                 const struct drm_display_mode *adjusted_mode =
2358                         &enabled->config->base.adjusted_mode;
2359                 const struct drm_framebuffer *fb =
2360                         enabled->base.primary->state->fb;
2361                 int clock = adjusted_mode->crtc_clock;
2362                 int htotal = adjusted_mode->crtc_htotal;
2363                 int hdisplay = enabled->config->pipe_src_w;
2364                 int cpp;
2365                 int entries;
2366
2367                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2368                         cpp = 4;
2369                 else
2370                         cpp = fb->format->cpp[0];
2371
2372                 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2373                                            sr_latency_ns / 100);
2374                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2375                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
2376                 srwm = wm_info->fifo_size - entries;
2377                 if (srwm < 0)
2378                         srwm = 1;
2379
2380                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2381                         I915_WRITE(FW_BLC_SELF,
2382                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2383                 else
2384                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
2385         }
2386
2387         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2388                       planea_wm, planeb_wm, cwm, srwm);
2389
2390         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2391         fwater_hi = (cwm & 0x1f);
2392
2393         /* Set request length to 8 cachelines per fetch */
2394         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2395         fwater_hi = fwater_hi | (1 << 8);
2396
2397         I915_WRITE(FW_BLC, fwater_lo);
2398         I915_WRITE(FW_BLC2, fwater_hi);
2399
2400         if (enabled)
2401                 intel_set_memory_cxsr(dev_priv, true);
2402 }
2403
2404 static void i845_update_wm(struct intel_crtc *unused_crtc)
2405 {
2406         struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
2407         struct intel_crtc *crtc;
2408         const struct drm_display_mode *adjusted_mode;
2409         uint32_t fwater_lo;
2410         int planea_wm;
2411
2412         crtc = single_enabled_crtc(dev_priv);
2413         if (crtc == NULL)
2414                 return;
2415
2416         adjusted_mode = &crtc->config->base.adjusted_mode;
2417         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
2418                                        &i845_wm_info,
2419                                        dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
2420                                        4, pessimal_latency_ns);
2421         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
2422         fwater_lo |= (3<<8) | planea_wm;
2423
2424         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
2425
2426         I915_WRITE(FW_BLC, fwater_lo);
2427 }
2428
2429 /* latency must be in 0.1us units. */
2430 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2431                                    unsigned int cpp,
2432                                    unsigned int latency)
2433 {
2434         unsigned int ret;
2435
2436         ret = intel_wm_method1(pixel_rate, cpp, latency);
2437         ret = DIV_ROUND_UP(ret, 64) + 2;
2438
2439         return ret;
2440 }
2441
2442 /* latency must be in 0.1us units. */
2443 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2444                                    unsigned int htotal,
2445                                    unsigned int width,
2446                                    unsigned int cpp,
2447                                    unsigned int latency)
2448 {
2449         unsigned int ret;
2450
2451         ret = intel_wm_method2(pixel_rate, htotal,
2452                                width, cpp, latency);
2453         ret = DIV_ROUND_UP(ret, 64) + 2;
2454
2455         return ret;
2456 }
2457
2458 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
2459                            uint8_t cpp)
2460 {
2461         /*
2462          * Neither of these should be possible since this function shouldn't be
2463          * called if the CRTC is off or the plane is invisible.  But let's be
2464          * extra paranoid to avoid a potential divide-by-zero if we screw up
2465          * elsewhere in the driver.
2466          */
2467         if (WARN_ON(!cpp))
2468                 return 0;
2469         if (WARN_ON(!horiz_pixels))
2470                 return 0;
2471
2472         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2473 }
2474
2475 struct ilk_wm_maximums {
2476         uint16_t pri;
2477         uint16_t spr;
2478         uint16_t cur;
2479         uint16_t fbc;
2480 };
2481
2482 /*
2483  * For both WM_PIPE and WM_LP.
2484  * mem_value must be in 0.1us units.
2485  */
2486 static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
2487                                    const struct intel_plane_state *pstate,
2488                                    uint32_t mem_value,
2489                                    bool is_lp)
2490 {
2491         uint32_t method1, method2;
2492         int cpp;
2493
2494         if (!intel_wm_plane_visible(cstate, pstate))
2495                 return 0;
2496
2497         cpp = pstate->base.fb->format->cpp[0];
2498
2499         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2500
2501         if (!is_lp)
2502                 return method1;
2503
2504         method2 = ilk_wm_method2(cstate->pixel_rate,
2505                                  cstate->base.adjusted_mode.crtc_htotal,
2506                                  drm_rect_width(&pstate->base.dst),
2507                                  cpp, mem_value);
2508
2509         return min(method1, method2);
2510 }
2511
2512 /*
2513  * For both WM_PIPE and WM_LP.
2514  * mem_value must be in 0.1us units.
2515  */
2516 static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
2517                                    const struct intel_plane_state *pstate,
2518                                    uint32_t mem_value)
2519 {
2520         uint32_t method1, method2;
2521         int cpp;
2522
2523         if (!intel_wm_plane_visible(cstate, pstate))
2524                 return 0;
2525
2526         cpp = pstate->base.fb->format->cpp[0];
2527
2528         method1 = ilk_wm_method1(cstate->pixel_rate, cpp, mem_value);
2529         method2 = ilk_wm_method2(cstate->pixel_rate,
2530                                  cstate->base.adjusted_mode.crtc_htotal,
2531                                  drm_rect_width(&pstate->base.dst),
2532                                  cpp, mem_value);
2533         return min(method1, method2);
2534 }
2535
2536 /*
2537  * For both WM_PIPE and WM_LP.
2538  * mem_value must be in 0.1us units.
2539  */
2540 static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
2541                                    const struct intel_plane_state *pstate,
2542                                    uint32_t mem_value)
2543 {
2544         int cpp;
2545
2546         if (!intel_wm_plane_visible(cstate, pstate))
2547                 return 0;
2548
2549         cpp = pstate->base.fb->format->cpp[0];
2550
2551         return ilk_wm_method2(cstate->pixel_rate,
2552                               cstate->base.adjusted_mode.crtc_htotal,
2553                               pstate->base.crtc_w, cpp, mem_value);
2554 }
2555
2556 /* Only for WM_LP. */
2557 static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
2558                                    const struct intel_plane_state *pstate,
2559                                    uint32_t pri_val)
2560 {
2561         int cpp;
2562
2563         if (!intel_wm_plane_visible(cstate, pstate))
2564                 return 0;
2565
2566         cpp = pstate->base.fb->format->cpp[0];
2567
2568         return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->base.dst), cpp);
2569 }
2570
2571 static unsigned int
2572 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2573 {
2574         if (INTEL_GEN(dev_priv) >= 8)
2575                 return 3072;
2576         else if (INTEL_GEN(dev_priv) >= 7)
2577                 return 768;
2578         else
2579                 return 512;
2580 }
2581
2582 static unsigned int
2583 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2584                      int level, bool is_sprite)
2585 {
2586         if (INTEL_GEN(dev_priv) >= 8)
2587                 /* BDW primary/sprite plane watermarks */
2588                 return level == 0 ? 255 : 2047;
2589         else if (INTEL_GEN(dev_priv) >= 7)
2590                 /* IVB/HSW primary/sprite plane watermarks */
2591                 return level == 0 ? 127 : 1023;
2592         else if (!is_sprite)
2593                 /* ILK/SNB primary plane watermarks */
2594                 return level == 0 ? 127 : 511;
2595         else
2596                 /* ILK/SNB sprite plane watermarks */
2597                 return level == 0 ? 63 : 255;
2598 }
2599
2600 static unsigned int
2601 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2602 {
2603         if (INTEL_GEN(dev_priv) >= 7)
2604                 return level == 0 ? 63 : 255;
2605         else
2606                 return level == 0 ? 31 : 63;
2607 }
2608
2609 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2610 {
2611         if (INTEL_GEN(dev_priv) >= 8)
2612                 return 31;
2613         else
2614                 return 15;
2615 }
2616
2617 /* Calculate the maximum primary/sprite plane watermark */
2618 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
2619                                      int level,
2620                                      const struct intel_wm_config *config,
2621                                      enum intel_ddb_partitioning ddb_partitioning,
2622                                      bool is_sprite)
2623 {
2624         struct drm_i915_private *dev_priv = to_i915(dev);
2625         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2626
2627         /* if sprites aren't enabled, sprites get nothing */
2628         if (is_sprite && !config->sprites_enabled)
2629                 return 0;
2630
2631         /* HSW allows LP1+ watermarks even with multiple pipes */
2632         if (level == 0 || config->num_pipes_active > 1) {
2633                 fifo_size /= INTEL_INFO(dev_priv)->num_pipes;
2634
2635                 /*
2636                  * For some reason the non self refresh
2637                  * FIFO size is only half of the self
2638                  * refresh FIFO size on ILK/SNB.
2639                  */
2640                 if (INTEL_GEN(dev_priv) <= 6)
2641                         fifo_size /= 2;
2642         }
2643
2644         if (config->sprites_enabled) {
2645                 /* level 0 is always calculated with 1:1 split */
2646                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2647                         if (is_sprite)
2648                                 fifo_size *= 5;
2649                         fifo_size /= 6;
2650                 } else {
2651                         fifo_size /= 2;
2652                 }
2653         }
2654
2655         /* clamp to max that the registers can hold */
2656         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2657 }
2658
2659 /* Calculate the maximum cursor plane watermark */
2660 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
2661                                       int level,
2662                                       const struct intel_wm_config *config)
2663 {
2664         /* HSW LP1+ watermarks w/ multiple pipes */
2665         if (level > 0 && config->num_pipes_active > 1)
2666                 return 64;
2667
2668         /* otherwise just report max that registers can hold */
2669         return ilk_cursor_wm_reg_max(to_i915(dev), level);
2670 }
2671
2672 static void ilk_compute_wm_maximums(const struct drm_device *dev,
2673                                     int level,
2674                                     const struct intel_wm_config *config,
2675                                     enum intel_ddb_partitioning ddb_partitioning,
2676                                     struct ilk_wm_maximums *max)
2677 {
2678         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2679         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2680         max->cur = ilk_cursor_wm_max(dev, level, config);
2681         max->fbc = ilk_fbc_wm_reg_max(to_i915(dev));
2682 }
2683
2684 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2685                                         int level,
2686                                         struct ilk_wm_maximums *max)
2687 {
2688         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2689         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2690         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2691         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2692 }
2693
2694 static bool ilk_validate_wm_level(int level,
2695                                   const struct ilk_wm_maximums *max,
2696                                   struct intel_wm_level *result)
2697 {
2698         bool ret;
2699
2700         /* already determined to be invalid? */
2701         if (!result->enable)
2702                 return false;
2703
2704         result->enable = result->pri_val <= max->pri &&
2705                          result->spr_val <= max->spr &&
2706                          result->cur_val <= max->cur;
2707
2708         ret = result->enable;
2709
2710         /*
2711          * HACK until we can pre-compute everything,
2712          * and thus fail gracefully if LP0 watermarks
2713          * are exceeded...
2714          */
2715         if (level == 0 && !result->enable) {
2716                 if (result->pri_val > max->pri)
2717                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2718                                       level, result->pri_val, max->pri);
2719                 if (result->spr_val > max->spr)
2720                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2721                                       level, result->spr_val, max->spr);
2722                 if (result->cur_val > max->cur)
2723                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2724                                       level, result->cur_val, max->cur);
2725
2726                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2727                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2728                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2729                 result->enable = true;
2730         }
2731
2732         return ret;
2733 }
2734
2735 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2736                                  const struct intel_crtc *intel_crtc,
2737                                  int level,
2738                                  struct intel_crtc_state *cstate,
2739                                  const struct intel_plane_state *pristate,
2740                                  const struct intel_plane_state *sprstate,
2741                                  const struct intel_plane_state *curstate,
2742                                  struct intel_wm_level *result)
2743 {
2744         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2745         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2746         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2747
2748         /* WM1+ latency values stored in 0.5us units */
2749         if (level > 0) {
2750                 pri_latency *= 5;
2751                 spr_latency *= 5;
2752                 cur_latency *= 5;
2753         }
2754
2755         if (pristate) {
2756                 result->pri_val = ilk_compute_pri_wm(cstate, pristate,
2757                                                      pri_latency, level);
2758                 result->fbc_val = ilk_compute_fbc_wm(cstate, pristate, result->pri_val);
2759         }
2760
2761         if (sprstate)
2762                 result->spr_val = ilk_compute_spr_wm(cstate, sprstate, spr_latency);
2763
2764         if (curstate)
2765                 result->cur_val = ilk_compute_cur_wm(cstate, curstate, cur_latency);
2766
2767         result->enable = true;
2768 }
2769
2770 static uint32_t
2771 hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
2772 {
2773         const struct intel_atomic_state *intel_state =
2774                 to_intel_atomic_state(cstate->base.state);
2775         const struct drm_display_mode *adjusted_mode =
2776                 &cstate->base.adjusted_mode;
2777         u32 linetime, ips_linetime;
2778
2779         if (!cstate->base.active)
2780                 return 0;
2781         if (WARN_ON(adjusted_mode->crtc_clock == 0))
2782                 return 0;
2783         if (WARN_ON(intel_state->cdclk.logical.cdclk == 0))
2784                 return 0;
2785
2786         /* The WM are computed with base on how long it takes to fill a single
2787          * row at the given clock rate, multiplied by 8.
2788          * */
2789         linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2790                                      adjusted_mode->crtc_clock);
2791         ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
2792                                          intel_state->cdclk.logical.cdclk);
2793
2794         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2795                PIPE_WM_LINETIME_TIME(linetime);
2796 }
2797
2798 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2799                                   uint16_t wm[8])
2800 {
2801         if (INTEL_GEN(dev_priv) >= 9) {
2802                 uint32_t val;
2803                 int ret, i;
2804                 int level, max_level = ilk_wm_max_level(dev_priv);
2805
2806                 /* read the first set of memory latencies[0:3] */
2807                 val = 0; /* data0 to be programmed to 0 for first set */
2808                 mutex_lock(&dev_priv->pcu_lock);
2809                 ret = sandybridge_pcode_read(dev_priv,
2810                                              GEN9_PCODE_READ_MEM_LATENCY,
2811                                              &val);
2812                 mutex_unlock(&dev_priv->pcu_lock);
2813
2814                 if (ret) {
2815                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2816                         return;
2817                 }
2818
2819                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2820                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2821                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2822                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2823                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2824                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2825                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2826
2827                 /* read the second set of memory latencies[4:7] */
2828                 val = 1; /* data0 to be programmed to 1 for second set */
2829                 mutex_lock(&dev_priv->pcu_lock);
2830                 ret = sandybridge_pcode_read(dev_priv,
2831                                              GEN9_PCODE_READ_MEM_LATENCY,
2832                                              &val);
2833                 mutex_unlock(&dev_priv->pcu_lock);
2834                 if (ret) {
2835                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
2836                         return;
2837                 }
2838
2839                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2840                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2841                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2842                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2843                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2844                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2845                                 GEN9_MEM_LATENCY_LEVEL_MASK;
2846
2847                 /*
2848                  * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2849                  * need to be disabled. We make sure to sanitize the values out
2850                  * of the punit to satisfy this requirement.
2851                  */
2852                 for (level = 1; level <= max_level; level++) {
2853                         if (wm[level] == 0) {
2854                                 for (i = level + 1; i <= max_level; i++)
2855                                         wm[i] = 0;
2856                                 break;
2857                         }
2858                 }
2859
2860                 /*
2861                  * WaWmMemoryReadLatency:skl+,glk
2862                  *
2863                  * punit doesn't take into account the read latency so we need
2864                  * to add 2us to the various latency levels we retrieve from the
2865                  * punit when level 0 response data us 0us.
2866                  */
2867                 if (wm[0] == 0) {
2868                         wm[0] += 2;
2869                         for (level = 1; level <= max_level; level++) {
2870                                 if (wm[level] == 0)
2871                                         break;
2872                                 wm[level] += 2;
2873                         }
2874                 }
2875
2876         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2877                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2878
2879                 wm[0] = (sskpd >> 56) & 0xFF;
2880                 if (wm[0] == 0)
2881                         wm[0] = sskpd & 0xF;
2882                 wm[1] = (sskpd >> 4) & 0xFF;
2883                 wm[2] = (sskpd >> 12) & 0xFF;
2884                 wm[3] = (sskpd >> 20) & 0x1FF;
2885                 wm[4] = (sskpd >> 32) & 0x1FF;
2886         } else if (INTEL_GEN(dev_priv) >= 6) {
2887                 uint32_t sskpd = I915_READ(MCH_SSKPD);
2888
2889                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2890                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2891                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2892                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2893         } else if (INTEL_GEN(dev_priv) >= 5) {
2894                 uint32_t mltr = I915_READ(MLTR_ILK);
2895
2896                 /* ILK primary LP0 latency is 700 ns */
2897                 wm[0] = 7;
2898                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2899                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2900         } else {
2901                 MISSING_CASE(INTEL_DEVID(dev_priv));
2902         }
2903 }
2904
2905 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2906                                        uint16_t wm[5])
2907 {
2908         /* ILK sprite LP0 latency is 1300 ns */
2909         if (IS_GEN5(dev_priv))
2910                 wm[0] = 13;
2911 }
2912
2913 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2914                                        uint16_t wm[5])
2915 {
2916         /* ILK cursor LP0 latency is 1300 ns */
2917         if (IS_GEN5(dev_priv))
2918                 wm[0] = 13;
2919
2920         /* WaDoubleCursorLP3Latency:ivb */
2921         if (IS_IVYBRIDGE(dev_priv))
2922                 wm[3] *= 2;
2923 }
2924
2925 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2926 {
2927         /* how many WM levels are we expecting */
2928         if (INTEL_GEN(dev_priv) >= 9)
2929                 return 7;
2930         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2931                 return 4;
2932         else if (INTEL_GEN(dev_priv) >= 6)
2933                 return 3;
2934         else
2935                 return 2;
2936 }
2937
2938 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2939                                    const char *name,
2940                                    const uint16_t wm[8])
2941 {
2942         int level, max_level = ilk_wm_max_level(dev_priv);
2943
2944         for (level = 0; level <= max_level; level++) {
2945                 unsigned int latency = wm[level];
2946
2947                 if (latency == 0) {
2948                         DRM_ERROR("%s WM%d latency not provided\n",
2949                                   name, level);
2950                         continue;
2951                 }
2952
2953                 /*
2954                  * - latencies are in us on gen9.
2955                  * - before then, WM1+ latency values are in 0.5us units
2956                  */
2957                 if (INTEL_GEN(dev_priv) >= 9)
2958                         latency *= 10;
2959                 else if (level > 0)
2960                         latency *= 5;
2961
2962                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2963                               name, level, wm[level],
2964                               latency / 10, latency % 10);
2965         }
2966 }
2967
2968 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2969                                     uint16_t wm[5], uint16_t min)
2970 {
2971         int level, max_level = ilk_wm_max_level(dev_priv);
2972
2973         if (wm[0] >= min)
2974                 return false;
2975
2976         wm[0] = max(wm[0], min);
2977         for (level = 1; level <= max_level; level++)
2978                 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
2979
2980         return true;
2981 }
2982
2983 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2984 {
2985         bool changed;
2986
2987         /*
2988          * The BIOS provided WM memory latency values are often
2989          * inadequate for high resolution displays. Adjust them.
2990          */
2991         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
2992                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
2993                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
2994
2995         if (!changed)
2996                 return;
2997
2998         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
2999         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3000         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3001         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3002 }
3003
3004 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3005 {
3006         intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3007
3008         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3009                sizeof(dev_priv->wm.pri_latency));
3010         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3011                sizeof(dev_priv->wm.pri_latency));
3012
3013         intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3014         intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3015
3016         intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3017         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3018         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3019
3020         if (IS_GEN6(dev_priv))
3021                 snb_wm_latency_quirk(dev_priv);
3022 }
3023
3024 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3025 {
3026         intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3027         intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3028 }
3029
3030 static bool ilk_validate_pipe_wm(struct drm_device *dev,
3031                                  struct intel_pipe_wm *pipe_wm)
3032 {
3033         /* LP0 watermark maximums depend on this pipe alone */
3034         const struct intel_wm_config config = {
3035                 .num_pipes_active = 1,
3036                 .sprites_enabled = pipe_wm->sprites_enabled,
3037                 .sprites_scaled = pipe_wm->sprites_scaled,
3038         };
3039         struct ilk_wm_maximums max;
3040
3041         /* LP0 watermarks always use 1/2 DDB partitioning */
3042         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
3043
3044         /* At least LP0 must be valid */
3045         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3046                 DRM_DEBUG_KMS("LP0 watermark invalid\n");
3047                 return false;
3048         }
3049
3050         return true;
3051 }
3052
3053 /* Compute new watermarks for the pipe */
3054 static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
3055 {
3056         struct drm_atomic_state *state = cstate->base.state;
3057         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
3058         struct intel_pipe_wm *pipe_wm;
3059         struct drm_device *dev = state->dev;
3060         const struct drm_i915_private *dev_priv = to_i915(dev);
3061         struct drm_plane *plane;
3062         const struct drm_plane_state *plane_state;
3063         const struct intel_plane_state *pristate = NULL;
3064         const struct intel_plane_state *sprstate = NULL;
3065         const struct intel_plane_state *curstate = NULL;
3066         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3067         struct ilk_wm_maximums max;
3068
3069         pipe_wm = &cstate->wm.ilk.optimal;
3070
3071         drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, &cstate->base) {
3072                 const struct intel_plane_state *ps = to_intel_plane_state(plane_state);
3073
3074                 if (plane->type == DRM_PLANE_TYPE_PRIMARY)
3075                         pristate = ps;
3076                 else if (plane->type == DRM_PLANE_TYPE_OVERLAY)
3077                         sprstate = ps;
3078                 else if (plane->type == DRM_PLANE_TYPE_CURSOR)
3079                         curstate = ps;
3080         }
3081
3082         pipe_wm->pipe_enabled = cstate->base.active;
3083         if (sprstate) {
3084                 pipe_wm->sprites_enabled = sprstate->base.visible;
3085                 pipe_wm->sprites_scaled = sprstate->base.visible &&
3086                         (drm_rect_width(&sprstate->base.dst) != drm_rect_width(&sprstate->base.src) >> 16 ||
3087                          drm_rect_height(&sprstate->base.dst) != drm_rect_height(&sprstate->base.src) >> 16);
3088         }
3089
3090         usable_level = max_level;
3091
3092         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3093         if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3094                 usable_level = 1;
3095
3096         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3097         if (pipe_wm->sprites_scaled)
3098                 usable_level = 0;
3099
3100         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3101         ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate,
3102                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3103
3104         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3105                 pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
3106
3107         if (!ilk_validate_pipe_wm(dev, pipe_wm))
3108                 return -EINVAL;
3109
3110         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3111
3112         for (level = 1; level <= usable_level; level++) {
3113                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3114
3115                 ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate,
3116                                      pristate, sprstate, curstate, wm);
3117
3118                 /*
3119                  * Disable any watermark level that exceeds the
3120                  * register maximums since such watermarks are
3121                  * always invalid.
3122                  */
3123                 if (!ilk_validate_wm_level(level, &max, wm)) {
3124                         memset(wm, 0, sizeof(*wm));
3125                         break;
3126                 }
3127         }
3128
3129         return 0;
3130 }
3131
3132 /*
3133  * Build a set of 'intermediate' watermark values that satisfy both the old
3134  * state and the new state.  These can be programmed to the hardware
3135  * immediately.
3136  */
3137 static int ilk_compute_intermediate_wm(struct drm_device *dev,
3138                                        struct intel_crtc *intel_crtc,
3139                                        struct intel_crtc_state *newstate)
3140 {
3141         struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
3142         struct intel_atomic_state *intel_state =
3143                 to_intel_atomic_state(newstate->base.state);
3144         const struct intel_crtc_state *oldstate =
3145                 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3146         const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
3147         int level, max_level = ilk_wm_max_level(to_i915(dev));
3148
3149         /*
3150          * Start with the final, target watermarks, then combine with the
3151          * currently active watermarks to get values that are safe both before
3152          * and after the vblank.
3153          */
3154         *a = newstate->wm.ilk.optimal;
3155         if (!newstate->base.active || drm_atomic_crtc_needs_modeset(&newstate->base))
3156                 return 0;
3157
3158         a->pipe_enabled |= b->pipe_enabled;
3159         a->sprites_enabled |= b->sprites_enabled;
3160         a->sprites_scaled |= b->sprites_scaled;
3161
3162         for (level = 0; level <= max_level; level++) {
3163                 struct intel_wm_level *a_wm = &a->wm[level];
3164                 const struct intel_wm_level *b_wm = &b->wm[level];
3165
3166                 a_wm->enable &= b_wm->enable;
3167                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3168                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3169                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3170                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3171         }
3172
3173         /*
3174          * We need to make sure that these merged watermark values are
3175          * actually a valid configuration themselves.  If they're not,
3176          * there's no safe way to transition from the old state to
3177          * the new state, so we need to fail the atomic transaction.
3178          */
3179         if (!ilk_validate_pipe_wm(dev, a))
3180                 return -EINVAL;
3181
3182         /*
3183          * If our intermediate WM are identical to the final WM, then we can
3184          * omit the post-vblank programming; only update if it's different.
3185          */
3186         if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3187                 newstate->wm.need_postvbl_update = true;
3188
3189         return 0;
3190 }
3191
3192 /*
3193  * Merge the watermarks from all active pipes for a specific level.
3194  */
3195 static void ilk_merge_wm_level(struct drm_device *dev,
3196                                int level,
3197                                struct intel_wm_level *ret_wm)
3198 {
3199         const struct intel_crtc *intel_crtc;
3200
3201         ret_wm->enable = true;
3202
3203         for_each_intel_crtc(dev, intel_crtc) {
3204                 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
3205                 const struct intel_wm_level *wm = &active->wm[level];
3206
3207                 if (!active->pipe_enabled)
3208                         continue;
3209
3210                 /*
3211                  * The watermark values may have been used in the past,
3212                  * so we must maintain them in the registers for some
3213                  * time even if the level is now disabled.
3214                  */
3215                 if (!wm->enable)
3216                         ret_wm->enable = false;
3217
3218                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3219                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3220                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3221                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3222         }
3223 }
3224
3225 /*
3226  * Merge all low power watermarks for all active pipes.
3227  */
3228 static void ilk_wm_merge(struct drm_device *dev,
3229                          const struct intel_wm_config *config,
3230                          const struct ilk_wm_maximums *max,
3231                          struct intel_pipe_wm *merged)
3232 {
3233         struct drm_i915_private *dev_priv = to_i915(dev);
3234         int level, max_level = ilk_wm_max_level(dev_priv);
3235         int last_enabled_level = max_level;
3236
3237         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3238         if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3239             config->num_pipes_active > 1)
3240                 last_enabled_level = 0;
3241
3242         /* ILK: FBC WM must be disabled always */
3243         merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
3244
3245         /* merge each WM1+ level */
3246         for (level = 1; level <= max_level; level++) {
3247                 struct intel_wm_level *wm = &merged->wm[level];
3248
3249                 ilk_merge_wm_level(dev, level, wm);
3250
3251                 if (level > last_enabled_level)
3252                         wm->enable = false;
3253                 else if (!ilk_validate_wm_level(level, max, wm))
3254                         /* make sure all following levels get disabled */
3255                         last_enabled_level = level - 1;
3256
3257                 /*
3258                  * The spec says it is preferred to disable
3259                  * FBC WMs instead of disabling a WM level.
3260                  */
3261                 if (wm->fbc_val > max->fbc) {
3262                         if (wm->enable)
3263                                 merged->fbc_wm_enabled = false;
3264                         wm->fbc_val = 0;
3265                 }
3266         }
3267
3268         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3269         /*
3270          * FIXME this is racy. FBC might get enabled later.
3271          * What we should check here is whether FBC can be
3272          * enabled sometime later.
3273          */
3274         if (IS_GEN5(dev_priv) && !merged->fbc_wm_enabled &&
3275             intel_fbc_is_active(dev_priv)) {
3276                 for (level = 2; level <= max_level; level++) {
3277                         struct intel_wm_level *wm = &merged->wm[level];
3278
3279                         wm->enable = false;
3280                 }
3281         }
3282 }
3283
3284 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3285 {
3286         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3287         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3288 }
3289
3290 /* The value we need to program into the WM_LPx latency field */
3291 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
3292 {
3293         struct drm_i915_private *dev_priv = to_i915(dev);
3294
3295         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3296                 return 2 * level;
3297         else
3298                 return dev_priv->wm.pri_latency[level];
3299 }
3300
3301 static void ilk_compute_wm_results(struct drm_device *dev,
3302                                    const struct intel_pipe_wm *merged,
3303                                    enum intel_ddb_partitioning partitioning,
3304                                    struct ilk_wm_values *results)
3305 {
3306         struct drm_i915_private *dev_priv = to_i915(dev);
3307         struct intel_crtc *intel_crtc;
3308         int level, wm_lp;
3309
3310         results->enable_fbc_wm = merged->fbc_wm_enabled;
3311         results->partitioning = partitioning;
3312
3313         /* LP1+ register values */
3314         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3315                 const struct intel_wm_level *r;
3316
3317                 level = ilk_wm_lp_to_level(wm_lp, merged);
3318
3319                 r = &merged->wm[level];
3320
3321                 /*
3322                  * Maintain the watermark values even if the level is
3323                  * disabled. Doing otherwise could cause underruns.
3324                  */
3325                 results->wm_lp[wm_lp - 1] =
3326                         (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
3327                         (r->pri_val << WM1_LP_SR_SHIFT) |
3328                         r->cur_val;
3329
3330                 if (r->enable)
3331                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3332
3333                 if (INTEL_GEN(dev_priv) >= 8)
3334                         results->wm_lp[wm_lp - 1] |=
3335                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3336                 else
3337                         results->wm_lp[wm_lp - 1] |=
3338                                 r->fbc_val << WM1_LP_FBC_SHIFT;
3339
3340                 /*
3341                  * Always set WM1S_LP_EN when spr_val != 0, even if the
3342                  * level is disabled. Doing otherwise could cause underruns.
3343                  */
3344                 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
3345                         WARN_ON(wm_lp != 1);
3346                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3347                 } else
3348                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
3349         }
3350
3351         /* LP0 register values */
3352         for_each_intel_crtc(dev, intel_crtc) {
3353                 enum pipe pipe = intel_crtc->pipe;
3354                 const struct intel_wm_level *r =
3355                         &intel_crtc->wm.active.ilk.wm[0];
3356
3357                 if (WARN_ON(!r->enable))
3358                         continue;
3359
3360                 results->wm_linetime[pipe] = intel_crtc->wm.active.ilk.linetime;
3361
3362                 results->wm_pipe[pipe] =
3363                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3364                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3365                         r->cur_val;
3366         }
3367 }
3368
3369 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3370  * case both are at the same level. Prefer r1 in case they're the same. */
3371 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
3372                                                   struct intel_pipe_wm *r1,
3373                                                   struct intel_pipe_wm *r2)
3374 {
3375         int level, max_level = ilk_wm_max_level(to_i915(dev));
3376         int level1 = 0, level2 = 0;
3377
3378         for (level = 1; level <= max_level; level++) {
3379                 if (r1->wm[level].enable)
3380                         level1 = level;
3381                 if (r2->wm[level].enable)
3382                         level2 = level;
3383         }
3384
3385         if (level1 == level2) {
3386                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3387                         return r2;
3388                 else
3389                         return r1;
3390         } else if (level1 > level2) {
3391                 return r1;
3392         } else {
3393                 return r2;
3394         }
3395 }
3396
3397 /* dirty bits used to track which watermarks need changes */
3398 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3399 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
3400 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3401 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3402 #define WM_DIRTY_FBC (1 << 24)
3403 #define WM_DIRTY_DDB (1 << 25)
3404
3405 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3406                                          const struct ilk_wm_values *old,
3407                                          const struct ilk_wm_values *new)
3408 {
3409         unsigned int dirty = 0;
3410         enum pipe pipe;
3411         int wm_lp;
3412
3413         for_each_pipe(dev_priv, pipe) {
3414                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
3415                         dirty |= WM_DIRTY_LINETIME(pipe);
3416                         /* Must disable LP1+ watermarks too */
3417                         dirty |= WM_DIRTY_LP_ALL;
3418                 }
3419
3420                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3421                         dirty |= WM_DIRTY_PIPE(pipe);
3422                         /* Must disable LP1+ watermarks too */
3423                         dirty |= WM_DIRTY_LP_ALL;
3424                 }
3425         }
3426
3427         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3428                 dirty |= WM_DIRTY_FBC;
3429                 /* Must disable LP1+ watermarks too */
3430                 dirty |= WM_DIRTY_LP_ALL;
3431         }
3432
3433         if (old->partitioning != new->partitioning) {
3434                 dirty |= WM_DIRTY_DDB;
3435                 /* Must disable LP1+ watermarks too */
3436                 dirty |= WM_DIRTY_LP_ALL;
3437         }
3438
3439         /* LP1+ watermarks already deemed dirty, no need to continue */
3440         if (dirty & WM_DIRTY_LP_ALL)
3441                 return dirty;
3442
3443         /* Find the lowest numbered LP1+ watermark in need of an update... */
3444         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3445                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3446                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3447                         break;
3448         }
3449
3450         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3451         for (; wm_lp <= 3; wm_lp++)
3452                 dirty |= WM_DIRTY_LP(wm_lp);
3453
3454         return dirty;
3455 }
3456
3457 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3458                                unsigned int dirty)
3459 {
3460         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3461         bool changed = false;
3462
3463         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3464                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
3465                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
3466                 changed = true;
3467         }
3468         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3469                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
3470                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
3471                 changed = true;
3472         }
3473         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3474                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
3475                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
3476                 changed = true;
3477         }
3478
3479         /*
3480          * Don't touch WM1S_LP_EN here.
3481          * Doing so could cause underruns.
3482          */
3483
3484         return changed;
3485 }
3486
3487 /*
3488  * The spec says we shouldn't write when we don't need, because every write
3489  * causes WMs to be re-evaluated, expending some power.
3490  */
3491 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3492                                 struct ilk_wm_values *results)
3493 {
3494         struct ilk_wm_values *previous = &dev_priv->wm.hw;
3495         unsigned int dirty;
3496         uint32_t val;
3497
3498         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3499         if (!dirty)
3500                 return;
3501
3502         _ilk_disable_lp_wm(dev_priv, dirty);
3503
3504         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3505                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
3506         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3507                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
3508         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3509                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
3510
3511         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
3512                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
3513         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
3514                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
3515         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
3516                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
3517
3518         if (dirty & WM_DIRTY_DDB) {
3519                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3520                         val = I915_READ(WM_MISC);
3521                         if (results->partitioning == INTEL_DDB_PART_1_2)
3522                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
3523                         else
3524                                 val |= WM_MISC_DATA_PARTITION_5_6;
3525                         I915_WRITE(WM_MISC, val);
3526                 } else {
3527                         val = I915_READ(DISP_ARB_CTL2);
3528                         if (results->partitioning == INTEL_DDB_PART_1_2)
3529                                 val &= ~DISP_DATA_PARTITION_5_6;
3530                         else
3531                                 val |= DISP_DATA_PARTITION_5_6;
3532                         I915_WRITE(DISP_ARB_CTL2, val);
3533                 }
3534         }
3535
3536         if (dirty & WM_DIRTY_FBC) {
3537                 val = I915_READ(DISP_ARB_CTL);
3538                 if (results->enable_fbc_wm)
3539                         val &= ~DISP_FBC_WM_DIS;
3540                 else
3541                         val |= DISP_FBC_WM_DIS;
3542                 I915_WRITE(DISP_ARB_CTL, val);
3543         }
3544
3545         if (dirty & WM_DIRTY_LP(1) &&
3546             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3547                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
3548
3549         if (INTEL_GEN(dev_priv) >= 7) {
3550                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3551                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
3552                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3553                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
3554         }
3555
3556         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3557                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
3558         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3559                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
3560         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3561                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3562
3563         dev_priv->wm.hw = *results;
3564 }
3565
3566 bool ilk_disable_lp_wm(struct drm_device *dev)
3567 {
3568         struct drm_i915_private *dev_priv = to_i915(dev);
3569
3570         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3571 }
3572
3573 /*
3574  * FIXME: We still don't have the proper code detect if we need to apply the WA,
3575  * so assume we'll always need it in order to avoid underruns.
3576  */
3577 static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
3578 {
3579         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3580
3581         if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
3582                 return true;
3583
3584         return false;
3585 }
3586
3587 static bool
3588 intel_has_sagv(struct drm_i915_private *dev_priv)
3589 {
3590         if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
3591             IS_CANNONLAKE(dev_priv))
3592                 return true;
3593
3594         if (IS_SKYLAKE(dev_priv) &&
3595             dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED)
3596                 return true;
3597
3598         return false;
3599 }
3600
3601 /*
3602  * SAGV dynamically adjusts the system agent voltage and clock frequencies
3603  * depending on power and performance requirements. The display engine access
3604  * to system memory is blocked during the adjustment time. Because of the
3605  * blocking time, having this enabled can cause full system hangs and/or pipe
3606  * underruns if we don't meet all of the following requirements:
3607  *
3608  *  - <= 1 pipe enabled
3609  *  - All planes can enable watermarks for latencies >= SAGV engine block time
3610  *  - We're not using an interlaced display configuration
3611  */
3612 int
3613 intel_enable_sagv(struct drm_i915_private *dev_priv)
3614 {
3615         int ret;
3616
3617         if (!intel_has_sagv(dev_priv))
3618                 return 0;
3619
3620         if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3621                 return 0;
3622
3623         DRM_DEBUG_KMS("Enabling the SAGV\n");
3624         mutex_lock(&dev_priv->pcu_lock);
3625
3626         ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3627                                       GEN9_SAGV_ENABLE);
3628
3629         /* We don't need to wait for the SAGV when enabling */
3630         mutex_unlock(&dev_priv->pcu_lock);
3631
3632         /*
3633          * Some skl systems, pre-release machines in particular,
3634          * don't actually have an SAGV.
3635          */
3636         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3637                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3638                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3639                 return 0;
3640         } else if (ret < 0) {
3641                 DRM_ERROR("Failed to enable the SAGV\n");
3642                 return ret;
3643         }
3644
3645         dev_priv->sagv_status = I915_SAGV_ENABLED;
3646         return 0;
3647 }
3648
3649 int
3650 intel_disable_sagv(struct drm_i915_private *dev_priv)
3651 {
3652         int ret;
3653
3654         if (!intel_has_sagv(dev_priv))
3655                 return 0;
3656
3657         if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3658                 return 0;
3659
3660         DRM_DEBUG_KMS("Disabling the SAGV\n");
3661         mutex_lock(&dev_priv->pcu_lock);
3662
3663         /* bspec says to keep retrying for at least 1 ms */
3664         ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3665                                 GEN9_SAGV_DISABLE,
3666                                 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3667                                 1);
3668         mutex_unlock(&dev_priv->pcu_lock);
3669
3670         /*
3671          * Some skl systems, pre-release machines in particular,
3672          * don't actually have an SAGV.
3673          */
3674         if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3675                 DRM_DEBUG_DRIVER("No SAGV found on system, ignoring\n");
3676                 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3677                 return 0;
3678         } else if (ret < 0) {
3679                 DRM_ERROR("Failed to disable the SAGV (%d)\n", ret);
3680                 return ret;
3681         }
3682
3683         dev_priv->sagv_status = I915_SAGV_DISABLED;
3684         return 0;
3685 }
3686
3687 bool intel_can_enable_sagv(struct drm_atomic_state *state)
3688 {
3689         struct drm_device *dev = state->dev;
3690         struct drm_i915_private *dev_priv = to_i915(dev);
3691         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3692         struct intel_crtc *crtc;
3693         struct intel_plane *plane;
3694         struct intel_crtc_state *cstate;
3695         enum pipe pipe;
3696         int level, latency;
3697         int sagv_block_time_us = IS_GEN9(dev_priv) ? 30 : 20;
3698
3699         if (!intel_has_sagv(dev_priv))
3700                 return false;
3701
3702         /*
3703          * SKL+ workaround: bspec recommends we disable the SAGV when we have
3704          * more then one pipe enabled
3705          *
3706          * If there are no active CRTCs, no additional checks need be performed
3707          */
3708         if (hweight32(intel_state->active_crtcs) == 0)
3709                 return true;
3710         else if (hweight32(intel_state->active_crtcs) > 1)
3711                 return false;
3712
3713         /* Since we're now guaranteed to only have one active CRTC... */
3714         pipe = ffs(intel_state->active_crtcs) - 1;
3715         crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
3716         cstate = to_intel_crtc_state(crtc->base.state);
3717
3718         if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3719                 return false;
3720
3721         for_each_intel_plane_on_crtc(dev, crtc, plane) {
3722                 struct skl_plane_wm *wm =
3723                         &cstate->wm.skl.optimal.planes[plane->id];
3724
3725                 /* Skip this plane if it's not enabled */
3726                 if (!wm->wm[0].plane_en)
3727                         continue;
3728
3729                 /* Find the highest enabled wm level for this plane */
3730                 for (level = ilk_wm_max_level(dev_priv);
3731                      !wm->wm[level].plane_en; --level)
3732                      { }
3733
3734                 latency = dev_priv->wm.skl_latency[level];
3735
3736                 if (skl_needs_memory_bw_wa(intel_state) &&
3737                     plane->base.state->fb->modifier ==
3738                     I915_FORMAT_MOD_X_TILED)
3739                         latency += 15;
3740
3741                 /*
3742                  * If any of the planes on this pipe don't enable wm levels that
3743                  * incur memory latencies higher than sagv_block_time_us we
3744                  * can't enable the SAGV.
3745                  */
3746                 if (latency < sagv_block_time_us)
3747                         return false;
3748         }
3749
3750         return true;
3751 }
3752
3753 static void
3754 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
3755                                    const struct intel_crtc_state *cstate,
3756                                    struct skl_ddb_entry *alloc, /* out */
3757                                    int *num_active /* out */)
3758 {
3759         struct drm_atomic_state *state = cstate->base.state;
3760         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
3761         struct drm_i915_private *dev_priv = to_i915(dev);
3762         struct drm_crtc *for_crtc = cstate->base.crtc;
3763         unsigned int pipe_size, ddb_size;
3764         int nth_active_pipe;
3765
3766         if (WARN_ON(!state) || !cstate->base.active) {
3767                 alloc->start = 0;
3768                 alloc->end = 0;
3769                 *num_active = hweight32(dev_priv->active_crtcs);
3770                 return;
3771         }
3772
3773         if (intel_state->active_pipe_changes)
3774                 *num_active = hweight32(intel_state->active_crtcs);
3775         else
3776                 *num_active = hweight32(dev_priv->active_crtcs);
3777
3778         ddb_size = INTEL_INFO(dev_priv)->ddb_size;
3779         WARN_ON(ddb_size == 0);
3780
3781         ddb_size -= 4; /* 4 blocks for bypass path allocation */
3782
3783         /*
3784          * If the state doesn't change the active CRTC's, then there's
3785          * no need to recalculate; the existing pipe allocation limits
3786          * should remain unchanged.  Note that we're safe from racing
3787          * commits since any racing commit that changes the active CRTC
3788          * list would need to grab _all_ crtc locks, including the one
3789          * we currently hold.
3790          */
3791         if (!intel_state->active_pipe_changes) {
3792                 /*
3793                  * alloc may be cleared by clear_intel_crtc_state,
3794                  * copy from old state to be sure
3795                  */
3796                 *alloc = to_intel_crtc_state(for_crtc->state)->wm.skl.ddb;
3797                 return;
3798         }
3799
3800         nth_active_pipe = hweight32(intel_state->active_crtcs &
3801                                     (drm_crtc_mask(for_crtc) - 1));
3802         pipe_size = ddb_size / hweight32(intel_state->active_crtcs);
3803         alloc->start = nth_active_pipe * ddb_size / *num_active;
3804         alloc->end = alloc->start + pipe_size;
3805 }
3806
3807 static unsigned int skl_cursor_allocation(int num_active)
3808 {
3809         if (num_active == 1)
3810                 return 32;
3811
3812         return 8;
3813 }
3814
3815 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
3816 {
3817         entry->start = reg & 0x3ff;
3818         entry->end = (reg >> 16) & 0x3ff;
3819         if (entry->end)
3820                 entry->end += 1;
3821 }
3822
3823 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
3824                           struct skl_ddb_allocation *ddb /* out */)
3825 {
3826         struct intel_crtc *crtc;
3827
3828         memset(ddb, 0, sizeof(*ddb));
3829
3830         for_each_intel_crtc(&dev_priv->drm, crtc) {
3831                 enum intel_display_power_domain power_domain;
3832                 enum plane_id plane_id;
3833                 enum pipe pipe = crtc->pipe;
3834
3835                 power_domain = POWER_DOMAIN_PIPE(pipe);
3836                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
3837                         continue;
3838
3839                 for_each_plane_id_on_crtc(crtc, plane_id) {
3840                         u32 val;
3841
3842                         if (plane_id != PLANE_CURSOR)
3843                                 val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
3844                         else
3845                                 val = I915_READ(CUR_BUF_CFG(pipe));
3846
3847                         skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane_id], val);
3848                 }
3849
3850                 intel_display_power_put(dev_priv, power_domain);
3851         }
3852 }
3853
3854 /*
3855  * Determines the downscale amount of a plane for the purposes of watermark calculations.
3856  * The bspec defines downscale amount as:
3857  *
3858  * """
3859  * Horizontal down scale amount = maximum[1, Horizontal source size /
3860  *                                           Horizontal destination size]
3861  * Vertical down scale amount = maximum[1, Vertical source size /
3862  *                                         Vertical destination size]
3863  * Total down scale amount = Horizontal down scale amount *
3864  *                           Vertical down scale amount
3865  * """
3866  *
3867  * Return value is provided in 16.16 fixed point form to retain fractional part.
3868  * Caller should take care of dividing & rounding off the value.
3869  */
3870 static uint_fixed_16_16_t
3871 skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
3872                            const struct intel_plane_state *pstate)
3873 {
3874         struct intel_plane *plane = to_intel_plane(pstate->base.plane);
3875         uint32_t src_w, src_h, dst_w, dst_h;
3876         uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
3877         uint_fixed_16_16_t downscale_h, downscale_w;
3878
3879         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
3880                 return u32_to_fixed16(0);
3881
3882         /* n.b., src is 16.16 fixed point, dst is whole integer */
3883         if (plane->id == PLANE_CURSOR) {
3884                 /*
3885                  * Cursors only support 0/180 degree rotation,
3886                  * hence no need to account for rotation here.
3887                  */
3888                 src_w = pstate->base.src_w >> 16;
3889                 src_h = pstate->base.src_h >> 16;
3890                 dst_w = pstate->base.crtc_w;
3891                 dst_h = pstate->base.crtc_h;
3892         } else {
3893                 /*
3894                  * Src coordinates are already rotated by 270 degrees for
3895                  * the 90/270 degree plane rotation cases (to match the
3896                  * GTT mapping), hence no need to account for rotation here.
3897                  */
3898                 src_w = drm_rect_width(&pstate->base.src) >> 16;
3899                 src_h = drm_rect_height(&pstate->base.src) >> 16;
3900                 dst_w = drm_rect_width(&pstate->base.dst);
3901                 dst_h = drm_rect_height(&pstate->base.dst);
3902         }
3903
3904         fp_w_ratio = div_fixed16(src_w, dst_w);
3905         fp_h_ratio = div_fixed16(src_h, dst_h);
3906         downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
3907         downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
3908
3909         return mul_fixed16(downscale_w, downscale_h);
3910 }
3911
3912 static uint_fixed_16_16_t
3913 skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
3914 {
3915         uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
3916
3917         if (!crtc_state->base.enable)
3918                 return pipe_downscale;
3919
3920         if (crtc_state->pch_pfit.enabled) {
3921                 uint32_t src_w, src_h, dst_w, dst_h;
3922                 uint32_t pfit_size = crtc_state->pch_pfit.size;
3923                 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
3924                 uint_fixed_16_16_t downscale_h, downscale_w;
3925
3926                 src_w = crtc_state->pipe_src_w;
3927                 src_h = crtc_state->pipe_src_h;
3928                 dst_w = pfit_size >> 16;
3929                 dst_h = pfit_size & 0xffff;
3930
3931                 if (!dst_w || !dst_h)
3932                         return pipe_downscale;
3933
3934                 fp_w_ratio = div_fixed16(src_w, dst_w);
3935                 fp_h_ratio = div_fixed16(src_h, dst_h);
3936                 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
3937                 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
3938
3939                 pipe_downscale = mul_fixed16(downscale_w, downscale_h);
3940         }
3941
3942         return pipe_downscale;
3943 }
3944
3945 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
3946                                   struct intel_crtc_state *cstate)
3947 {
3948         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3949         struct drm_crtc_state *crtc_state = &cstate->base;
3950         struct drm_atomic_state *state = crtc_state->state;
3951         struct drm_plane *plane;
3952         const struct drm_plane_state *pstate;
3953         struct intel_plane_state *intel_pstate;
3954         int crtc_clock, dotclk;
3955         uint32_t pipe_max_pixel_rate;
3956         uint_fixed_16_16_t pipe_downscale;
3957         uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
3958
3959         if (!cstate->base.enable)
3960                 return 0;
3961
3962         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
3963                 uint_fixed_16_16_t plane_downscale;
3964                 uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
3965                 int bpp;
3966
3967                 if (!intel_wm_plane_visible(cstate,
3968                                             to_intel_plane_state(pstate)))
3969                         continue;
3970
3971                 if (WARN_ON(!pstate->fb))
3972                         return -EINVAL;
3973
3974                 intel_pstate = to_intel_plane_state(pstate);
3975                 plane_downscale = skl_plane_downscale_amount(cstate,
3976                                                              intel_pstate);
3977                 bpp = pstate->fb->format->cpp[0] * 8;
3978                 if (bpp == 64)
3979                         plane_downscale = mul_fixed16(plane_downscale,
3980                                                       fp_9_div_8);
3981
3982                 max_downscale = max_fixed16(plane_downscale, max_downscale);
3983         }
3984         pipe_downscale = skl_pipe_downscale_amount(cstate);
3985
3986         pipe_downscale = mul_fixed16(pipe_downscale, max_downscale);
3987
3988         crtc_clock = crtc_state->adjusted_mode.crtc_clock;
3989         dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
3990
3991         if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
3992                 dotclk *= 2;
3993
3994         pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
3995
3996         if (pipe_max_pixel_rate < crtc_clock) {
3997                 DRM_DEBUG_KMS("Max supported pixel clock with scaling exceeded\n");
3998                 return -EINVAL;
3999         }
4000
4001         return 0;
4002 }
4003
4004 static unsigned int
4005 skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
4006                              const struct drm_plane_state *pstate,
4007                              int y)
4008 {
4009         struct intel_plane *plane = to_intel_plane(pstate->plane);
4010         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4011         uint32_t data_rate;
4012         uint32_t width = 0, height = 0;
4013         struct drm_framebuffer *fb;
4014         u32 format;
4015         uint_fixed_16_16_t down_scale_amount;
4016
4017         if (!intel_pstate->base.visible)
4018                 return 0;
4019
4020         fb = pstate->fb;
4021         format = fb->format->format;
4022
4023         if (plane->id == PLANE_CURSOR)
4024                 return 0;
4025         if (y && format != DRM_FORMAT_NV12)
4026                 return 0;
4027
4028         /*
4029          * Src coordinates are already rotated by 270 degrees for
4030          * the 90/270 degree plane rotation cases (to match the
4031          * GTT mapping), hence no need to account for rotation here.
4032          */
4033         width = drm_rect_width(&intel_pstate->base.src) >> 16;
4034         height = drm_rect_height(&intel_pstate->base.src) >> 16;
4035
4036         /* for planar format */
4037         if (format == DRM_FORMAT_NV12) {
4038                 if (y)  /* y-plane data rate */
4039                         data_rate = width * height *
4040                                 fb->format->cpp[0];
4041                 else    /* uv-plane data rate */
4042                         data_rate = (width / 2) * (height / 2) *
4043                                 fb->format->cpp[1];
4044         } else {
4045                 /* for packed formats */
4046                 data_rate = width * height * fb->format->cpp[0];
4047         }
4048
4049         down_scale_amount = skl_plane_downscale_amount(cstate, intel_pstate);
4050
4051         return mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4052 }
4053
4054 /*
4055  * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
4056  * a 8192x4096@32bpp framebuffer:
4057  *   3 * 4096 * 8192  * 4 < 2^32
4058  */
4059 static unsigned int
4060 skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate,
4061                                  unsigned *plane_data_rate,
4062                                  unsigned *plane_y_data_rate)
4063 {
4064         struct drm_crtc_state *cstate = &intel_cstate->base;
4065         struct drm_atomic_state *state = cstate->state;
4066         struct drm_plane *plane;
4067         const struct drm_plane_state *pstate;
4068         unsigned int total_data_rate = 0;
4069
4070         if (WARN_ON(!state))
4071                 return 0;
4072
4073         /* Calculate and cache data rate for each plane */
4074         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) {
4075                 enum plane_id plane_id = to_intel_plane(plane)->id;
4076                 unsigned int rate;
4077
4078                 /* packed/uv */
4079                 rate = skl_plane_relative_data_rate(intel_cstate,
4080                                                     pstate, 0);
4081                 plane_data_rate[plane_id] = rate;
4082
4083                 total_data_rate += rate;
4084
4085                 /* y-plane */
4086                 rate = skl_plane_relative_data_rate(intel_cstate,
4087                                                     pstate, 1);
4088                 plane_y_data_rate[plane_id] = rate;
4089
4090                 total_data_rate += rate;
4091         }
4092
4093         return total_data_rate;
4094 }
4095
4096 static uint16_t
4097 skl_ddb_min_alloc(const struct drm_plane_state *pstate,
4098                   const int y)
4099 {
4100         struct drm_framebuffer *fb = pstate->fb;
4101         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
4102         uint32_t src_w, src_h;
4103         uint32_t min_scanlines = 8;
4104         uint8_t plane_bpp;
4105
4106         if (WARN_ON(!fb))
4107                 return 0;
4108
4109         /* For packed formats, no y-plane, return 0 */
4110         if (y && fb->format->format != DRM_FORMAT_NV12)
4111                 return 0;
4112
4113         /* For Non Y-tile return 8-blocks */
4114         if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
4115             fb->modifier != I915_FORMAT_MOD_Yf_TILED &&
4116             fb->modifier != I915_FORMAT_MOD_Y_TILED_CCS &&
4117             fb->modifier != I915_FORMAT_MOD_Yf_TILED_CCS)
4118                 return 8;
4119
4120         /*
4121          * Src coordinates are already rotated by 270 degrees for
4122          * the 90/270 degree plane rotation cases (to match the
4123          * GTT mapping), hence no need to account for rotation here.
4124          */
4125         src_w = drm_rect_width(&intel_pstate->base.src) >> 16;
4126         src_h = drm_rect_height(&intel_pstate->base.src) >> 16;
4127
4128         /* Halve UV plane width and height for NV12 */
4129         if (fb->format->format == DRM_FORMAT_NV12 && !y) {
4130                 src_w /= 2;
4131                 src_h /= 2;
4132         }
4133
4134         if (fb->format->format == DRM_FORMAT_NV12 && !y)
4135                 plane_bpp = fb->format->cpp[1];
4136         else
4137                 plane_bpp = fb->format->cpp[0];
4138
4139         if (drm_rotation_90_or_270(pstate->rotation)) {
4140                 switch (plane_bpp) {
4141                 case 1:
4142                         min_scanlines = 32;
4143                         break;
4144                 case 2:
4145                         min_scanlines = 16;
4146                         break;
4147                 case 4:
4148                         min_scanlines = 8;
4149                         break;
4150                 case 8:
4151                         min_scanlines = 4;
4152                         break;
4153                 default:
4154                         WARN(1, "Unsupported pixel depth %u for rotation",
4155                              plane_bpp);
4156                         min_scanlines = 32;
4157                 }
4158         }
4159
4160         return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
4161 }
4162
4163 static void
4164 skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active,
4165                  uint16_t *minimum, uint16_t *y_minimum)
4166 {
4167         const struct drm_plane_state *pstate;
4168         struct drm_plane *plane;
4169
4170         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) {
4171                 enum plane_id plane_id = to_intel_plane(plane)->id;
4172
4173                 if (plane_id == PLANE_CURSOR)
4174                         continue;
4175
4176                 if (!pstate->visible)
4177                         continue;
4178
4179                 minimum[plane_id] = skl_ddb_min_alloc(pstate, 0);
4180                 y_minimum[plane_id] = skl_ddb_min_alloc(pstate, 1);
4181         }
4182
4183         minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active);
4184 }
4185
4186 static int
4187 skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
4188                       struct skl_ddb_allocation *ddb /* out */)
4189 {
4190         struct drm_atomic_state *state = cstate->base.state;
4191         struct drm_crtc *crtc = cstate->base.crtc;
4192         struct drm_device *dev = crtc->dev;
4193         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4194         enum pipe pipe = intel_crtc->pipe;
4195         struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
4196         uint16_t alloc_size, start;
4197         uint16_t minimum[I915_MAX_PLANES] = {};
4198         uint16_t y_minimum[I915_MAX_PLANES] = {};
4199         unsigned int total_data_rate;
4200         enum plane_id plane_id;
4201         int num_active;
4202         unsigned plane_data_rate[I915_MAX_PLANES] = {};
4203         unsigned plane_y_data_rate[I915_MAX_PLANES] = {};
4204         uint16_t total_min_blocks = 0;
4205
4206         /* Clear the partitioning for disabled planes. */
4207         memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
4208         memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe]));
4209
4210         if (WARN_ON(!state))
4211                 return 0;
4212
4213         if (!cstate->base.active) {
4214                 alloc->start = alloc->end = 0;
4215                 return 0;
4216         }
4217
4218         skl_ddb_get_pipe_allocation_limits(dev, cstate, alloc, &num_active);
4219         alloc_size = skl_ddb_entry_size(alloc);
4220         if (alloc_size == 0)
4221                 return 0;
4222
4223         skl_ddb_calc_min(cstate, num_active, minimum, y_minimum);
4224
4225         /*
4226          * 1. Allocate the mininum required blocks for each active plane
4227          * and allocate the cursor, it doesn't require extra allocation
4228          * proportional to the data rate.
4229          */
4230
4231         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4232                 total_min_blocks += minimum[plane_id];
4233                 total_min_blocks += y_minimum[plane_id];
4234         }
4235
4236         if (total_min_blocks > alloc_size) {
4237                 DRM_DEBUG_KMS("Requested display configuration exceeds system DDB limitations");
4238                 DRM_DEBUG_KMS("minimum required %d/%d\n", total_min_blocks,
4239                                                         alloc_size);
4240                 return -EINVAL;
4241         }
4242
4243         alloc_size -= total_min_blocks;
4244         ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR];
4245         ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
4246
4247         /*
4248          * 2. Distribute the remaining space in proportion to the amount of
4249          * data each plane needs to fetch from memory.
4250          *
4251          * FIXME: we may not allocate every single block here.
4252          */
4253         total_data_rate = skl_get_total_relative_data_rate(cstate,
4254                                                            plane_data_rate,
4255                                                            plane_y_data_rate);
4256         if (total_data_rate == 0)
4257                 return 0;
4258
4259         start = alloc->start;
4260         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
4261                 unsigned int data_rate, y_data_rate;
4262                 uint16_t plane_blocks, y_plane_blocks = 0;
4263
4264                 if (plane_id == PLANE_CURSOR)
4265                         continue;
4266
4267                 data_rate = plane_data_rate[plane_id];
4268
4269                 /*
4270                  * allocation for (packed formats) or (uv-plane part of planar format):
4271                  * promote the expression to 64 bits to avoid overflowing, the
4272                  * result is < available as data_rate / total_data_rate < 1
4273                  */
4274                 plane_blocks = minimum[plane_id];
4275                 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
4276                                         total_data_rate);
4277
4278                 /* Leave disabled planes at (0,0) */
4279                 if (data_rate) {
4280                         ddb->plane[pipe][plane_id].start = start;
4281                         ddb->plane[pipe][plane_id].end = start + plane_blocks;
4282                 }
4283
4284                 start += plane_blocks;
4285
4286                 /*
4287                  * allocation for y_plane part of planar format:
4288                  */
4289                 y_data_rate = plane_y_data_rate[plane_id];
4290
4291                 y_plane_blocks = y_minimum[plane_id];
4292                 y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
4293                                         total_data_rate);
4294
4295                 if (y_data_rate) {
4296                         ddb->y_plane[pipe][plane_id].start = start;
4297                         ddb->y_plane[pipe][plane_id].end = start + y_plane_blocks;
4298                 }
4299
4300                 start += y_plane_blocks;
4301         }
4302
4303         return 0;
4304 }
4305
4306 /*
4307  * The max latency should be 257 (max the punit can code is 255 and we add 2us
4308  * for the read latency) and cpp should always be <= 8, so that
4309  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4310  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4311 */
4312 static uint_fixed_16_16_t
4313 skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate,
4314                uint8_t cpp, uint32_t latency)
4315 {
4316         uint32_t wm_intermediate_val;
4317         uint_fixed_16_16_t ret;
4318
4319         if (latency == 0)
4320                 return FP_16_16_MAX;
4321
4322         wm_intermediate_val = latency * pixel_rate * cpp;
4323         ret = div_fixed16(wm_intermediate_val, 1000 * 512);
4324
4325         if (INTEL_GEN(dev_priv) >= 10)
4326                 ret = add_fixed16_u32(ret, 1);
4327
4328         return ret;
4329 }
4330
4331 static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
4332                         uint32_t pipe_htotal,
4333                         uint32_t latency,
4334                         uint_fixed_16_16_t plane_blocks_per_line)
4335 {
4336         uint32_t wm_intermediate_val;
4337         uint_fixed_16_16_t ret;
4338
4339         if (latency == 0)
4340                 return FP_16_16_MAX;
4341
4342         wm_intermediate_val = latency * pixel_rate;
4343         wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
4344                                            pipe_htotal * 1000);
4345         ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
4346         return ret;
4347 }
4348
4349 static uint_fixed_16_16_t
4350 intel_get_linetime_us(struct intel_crtc_state *cstate)
4351 {
4352         uint32_t pixel_rate;
4353         uint32_t crtc_htotal;
4354         uint_fixed_16_16_t linetime_us;
4355
4356         if (!cstate->base.active)
4357                 return u32_to_fixed16(0);
4358
4359         pixel_rate = cstate->pixel_rate;
4360
4361         if (WARN_ON(pixel_rate == 0))
4362                 return u32_to_fixed16(0);
4363
4364         crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
4365         linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
4366
4367         return linetime_us;
4368 }
4369
4370 static uint32_t
4371 skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
4372                               const struct intel_plane_state *pstate)
4373 {
4374         uint64_t adjusted_pixel_rate;
4375         uint_fixed_16_16_t downscale_amount;
4376
4377         /* Shouldn't reach here on disabled planes... */
4378         if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
4379                 return 0;
4380
4381         /*
4382          * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
4383          * with additional adjustments for plane-specific scaling.
4384          */
4385         adjusted_pixel_rate = cstate->pixel_rate;
4386         downscale_amount = skl_plane_downscale_amount(cstate, pstate);
4387
4388         return mul_round_up_u32_fixed16(adjusted_pixel_rate,
4389                                             downscale_amount);
4390 }
4391
4392 static int
4393 skl_compute_plane_wm_params(const struct drm_i915_private *dev_priv,
4394                             struct intel_crtc_state *cstate,
4395                             const struct intel_plane_state *intel_pstate,
4396                             struct skl_wm_params *wp)
4397 {
4398         struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
4399         const struct drm_plane_state *pstate = &intel_pstate->base;
4400         const struct drm_framebuffer *fb = pstate->fb;
4401         uint32_t interm_pbpl;
4402         struct intel_atomic_state *state =
4403                 to_intel_atomic_state(cstate->base.state);
4404         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4405
4406         if (!intel_wm_plane_visible(cstate, intel_pstate))
4407                 return 0;
4408
4409         wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
4410                       fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
4411                       fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4412                       fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4413         wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
4414         wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
4415                          fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
4416
4417         if (plane->id == PLANE_CURSOR) {
4418                 wp->width = intel_pstate->base.crtc_w;
4419         } else {
4420                 /*
4421                  * Src coordinates are already rotated by 270 degrees for
4422                  * the 90/270 degree plane rotation cases (to match the
4423                  * GTT mapping), hence no need to account for rotation here.
4424                  */
4425                 wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
4426         }
4427
4428         wp->cpp = (fb->format->format == DRM_FORMAT_NV12) ? fb->format->cpp[1] :
4429                                                             fb->format->cpp[0];
4430         wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
4431                                                              intel_pstate);
4432
4433         if (drm_rotation_90_or_270(pstate->rotation)) {
4434
4435                 switch (wp->cpp) {
4436                 case 1:
4437                         wp->y_min_scanlines = 16;
4438                         break;
4439                 case 2:
4440                         wp->y_min_scanlines = 8;
4441                         break;
4442                 case 4:
4443                         wp->y_min_scanlines = 4;
4444                         break;
4445                 default:
4446                         MISSING_CASE(wp->cpp);
4447                         return -EINVAL;
4448                 }
4449         } else {
4450                 wp->y_min_scanlines = 4;
4451         }
4452
4453         if (apply_memory_bw_wa)
4454                 wp->y_min_scanlines *= 2;
4455
4456         wp->plane_bytes_per_line = wp->width * wp->cpp;
4457         if (wp->y_tiled) {
4458                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
4459                                            wp->y_min_scanlines, 512);
4460
4461                 if (INTEL_GEN(dev_priv) >= 10)
4462                         interm_pbpl++;
4463
4464                 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
4465                                                         wp->y_min_scanlines);
4466         } else if (wp->x_tiled && IS_GEN9(dev_priv)) {
4467                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, 512);
4468                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4469         } else {
4470                 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, 512) + 1;
4471                 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
4472         }
4473
4474         wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
4475                                              wp->plane_blocks_per_line);
4476         wp->linetime_us = fixed16_to_u32_round_up(
4477                                         intel_get_linetime_us(cstate));
4478
4479         return 0;
4480 }
4481
4482 static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
4483                                 struct intel_crtc_state *cstate,
4484                                 const struct intel_plane_state *intel_pstate,
4485                                 uint16_t ddb_allocation,
4486                                 int level,
4487                                 const struct skl_wm_params *wp,
4488                                 uint16_t *out_blocks, /* out */
4489                                 uint8_t *out_lines, /* out */
4490                                 bool *enabled /* out */)
4491 {
4492         const struct drm_plane_state *pstate = &intel_pstate->base;
4493         uint32_t latency = dev_priv->wm.skl_latency[level];
4494         uint_fixed_16_16_t method1, method2;
4495         uint_fixed_16_16_t selected_result;
4496         uint32_t res_blocks, res_lines;
4497         struct intel_atomic_state *state =
4498                 to_intel_atomic_state(cstate->base.state);
4499         bool apply_memory_bw_wa = skl_needs_memory_bw_wa(state);
4500
4501         if (latency == 0 ||
4502             !intel_wm_plane_visible(cstate, intel_pstate)) {
4503                 *enabled = false;
4504                 return 0;
4505         }
4506
4507         /* Display WA #1141: kbl,cfl */
4508         if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
4509             IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) &&
4510             dev_priv->ipc_enabled)
4511                 latency += 4;
4512
4513         if (apply_memory_bw_wa && wp->x_tiled)
4514                 latency += 15;
4515
4516         method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
4517                                  wp->cpp, latency);
4518         method2 = skl_wm_method2(wp->plane_pixel_rate,
4519                                  cstate->base.adjusted_mode.crtc_htotal,
4520                                  latency,
4521                                  wp->plane_blocks_per_line);
4522
4523         if (wp->y_tiled) {
4524                 selected_result = max_fixed16(method2, wp->y_tile_minimum);
4525         } else {
4526                 if ((wp->cpp * cstate->base.adjusted_mode.crtc_htotal /
4527                      512 < 1) && (wp->plane_bytes_per_line / 512 < 1))
4528                         selected_result = method2;
4529                 else if (ddb_allocation >=
4530                          fixed16_to_u32_round_up(wp->plane_blocks_per_line))
4531                         selected_result = min_fixed16(method1, method2);
4532                 else if (latency >= wp->linetime_us)
4533                         selected_result = min_fixed16(method1, method2);
4534                 else
4535                         selected_result = method1;
4536         }
4537
4538         res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
4539         res_lines = div_round_up_fixed16(selected_result,
4540                                          wp->plane_blocks_per_line);
4541
4542         /* Display WA #1125: skl,bxt,kbl,glk */
4543         if (level == 0 && wp->rc_surface)
4544                 res_blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
4545
4546         /* Display WA #1126: skl,bxt,kbl,glk */
4547         if (level >= 1 && level <= 7) {
4548                 if (wp->y_tiled) {
4549                         res_blocks += fixed16_to_u32_round_up(
4550                                                         wp->y_tile_minimum);
4551                         res_lines += wp->y_min_scanlines;
4552                 } else {
4553                         res_blocks++;
4554                 }
4555         }
4556
4557         if (res_blocks >= ddb_allocation || res_lines > 31) {
4558                 *enabled = false;
4559
4560                 /*
4561                  * If there are no valid level 0 watermarks, then we can't
4562                  * support this display configuration.
4563                  */
4564                 if (level) {
4565                         return 0;
4566                 } else {
4567                         struct drm_plane *plane = pstate->plane;
4568
4569                         DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
4570                         DRM_DEBUG_KMS("[PLANE:%d:%s] blocks required = %u/%u, lines required = %u/31\n",
4571                                       plane->base.id, plane->name,
4572                                       res_blocks, ddb_allocation, res_lines);
4573                         return -EINVAL;
4574                 }
4575         }
4576
4577         *out_blocks = res_blocks;
4578         *out_lines = res_lines;
4579         *enabled = true;
4580
4581         return 0;
4582 }
4583
4584 static int
4585 skl_compute_wm_levels(const struct drm_i915_private *dev_priv,
4586                       struct skl_ddb_allocation *ddb,
4587                       struct intel_crtc_state *cstate,
4588                       const struct intel_plane_state *intel_pstate,
4589                       const struct skl_wm_params *wm_params,
4590                       struct skl_plane_wm *wm)
4591 {
4592         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
4593         struct drm_plane *plane = intel_pstate->base.plane;
4594         struct intel_plane *intel_plane = to_intel_plane(plane);
4595         uint16_t ddb_blocks;
4596         enum pipe pipe = intel_crtc->pipe;
4597         int level, max_level = ilk_wm_max_level(dev_priv);
4598         int ret;
4599
4600         if (WARN_ON(!intel_pstate->base.fb))
4601                 return -EINVAL;
4602
4603         ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][intel_plane->id]);
4604
4605         for (level = 0; level <= max_level; level++) {
4606                 struct skl_wm_level *result = &wm->wm[level];
4607
4608                 ret = skl_compute_plane_wm(dev_priv,
4609                                            cstate,
4610                                            intel_pstate,
4611                                            ddb_blocks,
4612                                            level,
4613                                            wm_params,
4614                                            &result->plane_res_b,
4615                                            &result->plane_res_l,
4616                                            &result->plane_en);
4617                 if (ret)
4618                         return ret;
4619         }
4620
4621         return 0;
4622 }
4623
4624 static uint32_t
4625 skl_compute_linetime_wm(struct intel_crtc_state *cstate)
4626 {
4627         struct drm_atomic_state *state = cstate->base.state;
4628         struct drm_i915_private *dev_priv = to_i915(state->dev);
4629         uint_fixed_16_16_t linetime_us;
4630         uint32_t linetime_wm;
4631
4632         linetime_us = intel_get_linetime_us(cstate);
4633
4634         if (is_fixed16_zero(linetime_us))
4635                 return 0;
4636
4637         linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
4638
4639         /* Display WA #1135: bxt:ALL GLK:ALL */
4640         if ((IS_BROXTON(dev_priv) || IS_GEMINILAKE(dev_priv)) &&
4641             dev_priv->ipc_enabled)
4642                 linetime_wm /= 2;
4643
4644         return linetime_wm;
4645 }
4646
4647 static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
4648                                       struct skl_wm_params *wp,
4649                                       struct skl_wm_level *wm_l0,
4650                                       uint16_t ddb_allocation,
4651                                       struct skl_wm_level *trans_wm /* out */)
4652 {
4653         struct drm_device *dev = cstate->base.crtc->dev;
4654         const struct drm_i915_private *dev_priv = to_i915(dev);
4655         uint16_t trans_min, trans_y_tile_min;
4656         const uint16_t trans_amount = 10; /* This is configurable amount */
4657         uint16_t trans_offset_b, res_blocks;
4658
4659         if (!cstate->base.active)
4660                 goto exit;
4661
4662         /* Transition WM are not recommended by HW team for GEN9 */
4663         if (INTEL_GEN(dev_priv) <= 9)
4664                 goto exit;
4665
4666         /* Transition WM don't make any sense if ipc is disabled */
4667         if (!dev_priv->ipc_enabled)
4668                 goto exit;
4669
4670         if (INTEL_GEN(dev_priv) >= 10)
4671                 trans_min = 4;
4672
4673         trans_offset_b = trans_min + trans_amount;
4674
4675         if (wp->y_tiled) {
4676                 trans_y_tile_min = (uint16_t) mul_round_up_u32_fixed16(2,
4677                                                         wp->y_tile_minimum);
4678                 res_blocks = max(wm_l0->plane_res_b, trans_y_tile_min) +
4679                                 trans_offset_b;
4680         } else {
4681                 res_blocks = wm_l0->plane_res_b + trans_offset_b;
4682
4683                 /* WA BUG:1938466 add one block for non y-tile planes */
4684                 if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0))
4685                         res_blocks += 1;
4686
4687         }
4688
4689         res_blocks += 1;
4690
4691         if (res_blocks < ddb_allocation) {
4692                 trans_wm->plane_res_b = res_blocks;
4693                 trans_wm->plane_en = true;
4694                 return;
4695         }
4696
4697 exit:
4698         trans_wm->plane_en = false;
4699 }
4700
4701 static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
4702                              struct skl_ddb_allocation *ddb,
4703                              struct skl_pipe_wm *pipe_wm)
4704 {
4705         struct drm_device *dev = cstate->base.crtc->dev;
4706         struct drm_crtc_state *crtc_state = &cstate->base;
4707         const struct drm_i915_private *dev_priv = to_i915(dev);
4708         struct drm_plane *plane;
4709         const struct drm_plane_state *pstate;
4710         struct skl_plane_wm *wm;
4711         int ret;
4712
4713         /*
4714          * We'll only calculate watermarks for planes that are actually
4715          * enabled, so make sure all other planes are set as disabled.
4716          */
4717         memset(pipe_wm->planes, 0, sizeof(pipe_wm->planes));
4718
4719         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
4720                 const struct intel_plane_state *intel_pstate =
4721                                                 to_intel_plane_state(pstate);
4722                 enum plane_id plane_id = to_intel_plane(plane)->id;
4723                 struct skl_wm_params wm_params;
4724                 enum pipe pipe = to_intel_crtc(cstate->base.crtc)->pipe;
4725                 uint16_t ddb_blocks;
4726
4727                 wm = &pipe_wm->planes[plane_id];
4728                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][plane_id]);
4729                 memset(&wm_params, 0, sizeof(struct skl_wm_params));
4730
4731                 ret = skl_compute_plane_wm_params(dev_priv, cstate,
4732                                                   intel_pstate, &wm_params);
4733                 if (ret)
4734                         return ret;
4735
4736                 ret = skl_compute_wm_levels(dev_priv, ddb, cstate,
4737                                             intel_pstate, &wm_params, wm);
4738                 if (ret)
4739                         return ret;
4740                 skl_compute_transition_wm(cstate, &wm_params, &wm->wm[0],
4741                                           ddb_blocks, &wm->trans_wm);
4742         }
4743         pipe_wm->linetime = skl_compute_linetime_wm(cstate);
4744
4745         return 0;
4746 }
4747
4748 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
4749                                 i915_reg_t reg,
4750                                 const struct skl_ddb_entry *entry)
4751 {
4752         if (entry->end)
4753                 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
4754         else
4755                 I915_WRITE(reg, 0);
4756 }
4757
4758 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
4759                                i915_reg_t reg,
4760                                const struct skl_wm_level *level)
4761 {
4762         uint32_t val = 0;
4763
4764         if (level->plane_en) {
4765                 val |= PLANE_WM_EN;
4766                 val |= level->plane_res_b;
4767                 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
4768         }
4769
4770         I915_WRITE(reg, val);
4771 }
4772
4773 static void skl_write_plane_wm(struct intel_crtc *intel_crtc,
4774                                const struct skl_plane_wm *wm,
4775                                const struct skl_ddb_allocation *ddb,
4776                                enum plane_id plane_id)
4777 {
4778         struct drm_crtc *crtc = &intel_crtc->base;
4779         struct drm_device *dev = crtc->dev;
4780         struct drm_i915_private *dev_priv = to_i915(dev);
4781         int level, max_level = ilk_wm_max_level(dev_priv);
4782         enum pipe pipe = intel_crtc->pipe;
4783
4784         for (level = 0; level <= max_level; level++) {
4785                 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
4786                                    &wm->wm[level]);
4787         }
4788         skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
4789                            &wm->trans_wm);
4790
4791         skl_ddb_entry_write(dev_priv, PLANE_BUF_CFG(pipe, plane_id),
4792                             &ddb->plane[pipe][plane_id]);
4793         skl_ddb_entry_write(dev_priv, PLANE_NV12_BUF_CFG(pipe, plane_id),
4794                             &ddb->y_plane[pipe][plane_id]);
4795 }
4796
4797 static void skl_write_cursor_wm(struct intel_crtc *intel_crtc,
4798                                 const struct skl_plane_wm *wm,
4799                                 const struct skl_ddb_allocation *ddb)
4800 {
4801         struct drm_crtc *crtc = &intel_crtc->base;
4802         struct drm_device *dev = crtc->dev;
4803         struct drm_i915_private *dev_priv = to_i915(dev);
4804         int level, max_level = ilk_wm_max_level(dev_priv);
4805         enum pipe pipe = intel_crtc->pipe;
4806
4807         for (level = 0; level <= max_level; level++) {
4808                 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
4809                                    &wm->wm[level]);
4810         }
4811         skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm);
4812
4813         skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
4814                             &ddb->plane[pipe][PLANE_CURSOR]);
4815 }
4816
4817 bool skl_wm_level_equals(const struct skl_wm_level *l1,
4818                          const struct skl_wm_level *l2)
4819 {
4820         if (l1->plane_en != l2->plane_en)
4821                 return false;
4822
4823         /* If both planes aren't enabled, the rest shouldn't matter */
4824         if (!l1->plane_en)
4825                 return true;
4826
4827         return (l1->plane_res_l == l2->plane_res_l &&
4828                 l1->plane_res_b == l2->plane_res_b);
4829 }
4830
4831 static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
4832                                            const struct skl_ddb_entry *b)
4833 {
4834         return a->start < b->end && b->start < a->end;
4835 }
4836
4837 bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv,
4838                                  const struct skl_ddb_entry **entries,
4839                                  const struct skl_ddb_entry *ddb,
4840                                  int ignore)
4841 {
4842         enum pipe pipe;
4843
4844         for_each_pipe(dev_priv, pipe) {
4845                 if (pipe != ignore && entries[pipe] &&
4846                     skl_ddb_entries_overlap(ddb, entries[pipe]))
4847                         return true;
4848         }
4849
4850         return false;
4851 }
4852
4853 static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
4854                               const struct skl_pipe_wm *old_pipe_wm,
4855                               struct skl_pipe_wm *pipe_wm, /* out */
4856                               struct skl_ddb_allocation *ddb, /* out */
4857                               bool *changed /* out */)
4858 {
4859         struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
4860         int ret;
4861
4862         ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
4863         if (ret)
4864                 return ret;
4865
4866         if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm)))
4867                 *changed = false;
4868         else
4869                 *changed = true;
4870
4871         return 0;
4872 }
4873
4874 static uint32_t
4875 pipes_modified(struct drm_atomic_state *state)
4876 {
4877         struct drm_crtc *crtc;
4878         struct drm_crtc_state *cstate;
4879         uint32_t i, ret = 0;
4880
4881         for_each_new_crtc_in_state(state, crtc, cstate, i)
4882                 ret |= drm_crtc_mask(crtc);
4883
4884         return ret;
4885 }
4886
4887 static int
4888 skl_ddb_add_affected_planes(struct intel_crtc_state *cstate)
4889 {
4890         struct drm_atomic_state *state = cstate->base.state;
4891         struct drm_device *dev = state->dev;
4892         struct drm_crtc *crtc = cstate->base.crtc;
4893         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4894         struct drm_i915_private *dev_priv = to_i915(dev);
4895         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4896         struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
4897         struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
4898         struct drm_plane_state *plane_state;
4899         struct drm_plane *plane;
4900         enum pipe pipe = intel_crtc->pipe;
4901
4902         WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc));
4903
4904         drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) {
4905                 enum plane_id plane_id = to_intel_plane(plane)->id;
4906
4907                 if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][plane_id],
4908                                         &new_ddb->plane[pipe][plane_id]) &&
4909                     skl_ddb_entry_equal(&cur_ddb->y_plane[pipe][plane_id],
4910                                         &new_ddb->y_plane[pipe][plane_id]))
4911                         continue;
4912
4913                 plane_state = drm_atomic_get_plane_state(state, plane);
4914                 if (IS_ERR(plane_state))
4915                         return PTR_ERR(plane_state);
4916         }
4917
4918         return 0;
4919 }
4920
4921 static int
4922 skl_compute_ddb(struct drm_atomic_state *state)
4923 {
4924         struct drm_device *dev = state->dev;
4925         struct drm_i915_private *dev_priv = to_i915(dev);
4926         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
4927         struct intel_crtc *intel_crtc;
4928         struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
4929         uint32_t realloc_pipes = pipes_modified(state);
4930         int ret;
4931
4932         /*
4933          * If this is our first atomic update following hardware readout,
4934          * we can't trust the DDB that the BIOS programmed for us.  Let's
4935          * pretend that all pipes switched active status so that we'll
4936          * ensure a full DDB recompute.
4937          */
4938         if (dev_priv->wm.distrust_bios_wm) {
4939                 ret = drm_modeset_lock(&dev->mode_config.connection_mutex,
4940                                        state->acquire_ctx);
4941                 if (ret)
4942                         return ret;
4943
4944                 intel_state->active_pipe_changes = ~0;
4945
4946                 /*
4947                  * We usually only initialize intel_state->active_crtcs if we
4948                  * we're doing a modeset; make sure this field is always
4949                  * initialized during the sanitization process that happens
4950                  * on the first commit too.
4951                  */
4952                 if (!intel_state->modeset)
4953                         intel_state->active_crtcs = dev_priv->active_crtcs;
4954         }
4955
4956         /*
4957          * If the modeset changes which CRTC's are active, we need to
4958          * recompute the DDB allocation for *all* active pipes, even
4959          * those that weren't otherwise being modified in any way by this
4960          * atomic commit.  Due to the shrinking of the per-pipe allocations
4961          * when new active CRTC's are added, it's possible for a pipe that
4962          * we were already using and aren't changing at all here to suddenly
4963          * become invalid if its DDB needs exceeds its new allocation.
4964          *
4965          * Note that if we wind up doing a full DDB recompute, we can't let
4966          * any other display updates race with this transaction, so we need
4967          * to grab the lock on *all* CRTC's.
4968          */
4969         if (intel_state->active_pipe_changes) {
4970                 realloc_pipes = ~0;
4971                 intel_state->wm_results.dirty_pipes = ~0;
4972         }
4973
4974         /*
4975          * We're not recomputing for the pipes not included in the commit, so
4976          * make sure we start with the current state.
4977          */
4978         memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb));
4979
4980         for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
4981                 struct intel_crtc_state *cstate;
4982
4983                 cstate = intel_atomic_get_crtc_state(state, intel_crtc);
4984                 if (IS_ERR(cstate))
4985                         return PTR_ERR(cstate);
4986
4987                 ret = skl_allocate_pipe_ddb(cstate, ddb);
4988                 if (ret)
4989                         return ret;
4990
4991                 ret = skl_ddb_add_affected_planes(cstate);
4992                 if (ret)
4993                         return ret;
4994         }
4995
4996         return 0;
4997 }
4998
4999 static void
5000 skl_copy_wm_for_pipe(struct skl_wm_values *dst,
5001                      struct skl_wm_values *src,
5002                      enum pipe pipe)
5003 {
5004         memcpy(dst->ddb.y_plane[pipe], src->ddb.y_plane[pipe],
5005                sizeof(dst->ddb.y_plane[pipe]));
5006         memcpy(dst->ddb.plane[pipe], src->ddb.plane[pipe],
5007                sizeof(dst->ddb.plane[pipe]));
5008 }
5009
5010 static void
5011 skl_print_wm_changes(const struct drm_atomic_state *state)
5012 {
5013         const struct drm_device *dev = state->dev;
5014         const struct drm_i915_private *dev_priv = to_i915(dev);
5015         const struct intel_atomic_state *intel_state =
5016                 to_intel_atomic_state(state);
5017         const struct drm_crtc *crtc;
5018         const struct drm_crtc_state *cstate;
5019         const struct intel_plane *intel_plane;
5020         const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb;
5021         const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb;
5022         int i;
5023
5024         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5025                 const struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5026                 enum pipe pipe = intel_crtc->pipe;
5027
5028                 for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
5029                         enum plane_id plane_id = intel_plane->id;
5030                         const struct skl_ddb_entry *old, *new;
5031
5032                         old = &old_ddb->plane[pipe][plane_id];
5033                         new = &new_ddb->plane[pipe][plane_id];
5034
5035                         if (skl_ddb_entry_equal(old, new))
5036                                 continue;
5037
5038                         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
5039                                          intel_plane->base.base.id,
5040                                          intel_plane->base.name,
5041                                          old->start, old->end,
5042                                          new->start, new->end);
5043                 }
5044         }
5045 }
5046
5047 static int
5048 skl_compute_wm(struct drm_atomic_state *state)
5049 {
5050         struct drm_crtc *crtc;
5051         struct drm_crtc_state *cstate;
5052         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
5053         struct skl_wm_values *results = &intel_state->wm_results;
5054         struct drm_device *dev = state->dev;
5055         struct skl_pipe_wm *pipe_wm;
5056         bool changed = false;
5057         int ret, i;
5058
5059         /*
5060          * When we distrust bios wm we always need to recompute to set the
5061          * expected DDB allocations for each CRTC.
5062          */
5063         if (to_i915(dev)->wm.distrust_bios_wm)
5064                 changed = true;
5065
5066         /*
5067          * If this transaction isn't actually touching any CRTC's, don't
5068          * bother with watermark calculation.  Note that if we pass this
5069          * test, we're guaranteed to hold at least one CRTC state mutex,
5070          * which means we can safely use values like dev_priv->active_crtcs
5071          * since any racing commits that want to update them would need to
5072          * hold _all_ CRTC state mutexes.
5073          */
5074         for_each_new_crtc_in_state(state, crtc, cstate, i)
5075                 changed = true;
5076
5077         if (!changed)
5078                 return 0;
5079
5080         /* Clear all dirty flags */
5081         results->dirty_pipes = 0;
5082
5083         ret = skl_compute_ddb(state);
5084         if (ret)
5085                 return ret;
5086
5087         /*
5088          * Calculate WM's for all pipes that are part of this transaction.
5089          * Note that the DDB allocation above may have added more CRTC's that
5090          * weren't otherwise being modified (and set bits in dirty_pipes) if
5091          * pipe allocations had to change.
5092          *
5093          * FIXME:  Now that we're doing this in the atomic check phase, we
5094          * should allow skl_update_pipe_wm() to return failure in cases where
5095          * no suitable watermark values can be found.
5096          */
5097         for_each_new_crtc_in_state(state, crtc, cstate, i) {
5098                 struct intel_crtc_state *intel_cstate =
5099                         to_intel_crtc_state(cstate);
5100                 const struct skl_pipe_wm *old_pipe_wm =
5101                         &to_intel_crtc_state(crtc->state)->wm.skl.optimal;
5102
5103                 pipe_wm = &intel_cstate->wm.skl.optimal;
5104                 ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm,
5105                                          &results->ddb, &changed);
5106                 if (ret)
5107                         return ret;
5108
5109                 if (changed)
5110                         results->dirty_pipes |= drm_crtc_mask(crtc);
5111
5112                 if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
5113                         /* This pipe's WM's did not change */
5114                         continue;
5115
5116                 intel_cstate->update_wm_pre = true;
5117         }
5118
5119         skl_print_wm_changes(state);
5120
5121         return 0;
5122 }
5123
5124 static void skl_atomic_update_crtc_wm(struct intel_atomic_state *state,
5125                                       struct intel_crtc_state *cstate)
5126 {
5127         struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
5128         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5129         struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
5130         const struct skl_ddb_allocation *ddb = &state->wm_results.ddb;
5131         enum pipe pipe = crtc->pipe;
5132         enum plane_id plane_id;
5133
5134         if (!(state->wm_results.dirty_pipes & drm_crtc_mask(&crtc->base)))
5135                 return;
5136
5137         I915_WRITE(PIPE_WM_LINETIME(pipe), pipe_wm->linetime);
5138
5139         for_each_plane_id_on_crtc(crtc, plane_id) {
5140                 if (plane_id != PLANE_CURSOR)
5141                         skl_write_plane_wm(crtc, &pipe_wm->planes[plane_id],
5142                                            ddb, plane_id);
5143                 else
5144                         skl_write_cursor_wm(crtc, &pipe_wm->planes[plane_id],
5145                                             ddb);
5146         }
5147 }
5148
5149 static void skl_initial_wm(struct intel_atomic_state *state,
5150                            struct intel_crtc_state *cstate)
5151 {
5152         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5153         struct drm_device *dev = intel_crtc->base.dev;
5154         struct drm_i915_private *dev_priv = to_i915(dev);
5155         struct skl_wm_values *results = &state->wm_results;
5156         struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw;
5157         enum pipe pipe = intel_crtc->pipe;
5158
5159         if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0)
5160                 return;
5161
5162         mutex_lock(&dev_priv->wm.wm_mutex);
5163
5164         if (cstate->base.active_changed)
5165                 skl_atomic_update_crtc_wm(state, cstate);
5166
5167         skl_copy_wm_for_pipe(hw_vals, results, pipe);
5168
5169         mutex_unlock(&dev_priv->wm.wm_mutex);
5170 }
5171
5172 static void ilk_compute_wm_config(struct drm_device *dev,
5173                                   struct intel_wm_config *config)
5174 {
5175         struct intel_crtc *crtc;
5176
5177         /* Compute the currently _active_ config */
5178         for_each_intel_crtc(dev, crtc) {
5179                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
5180
5181                 if (!wm->pipe_enabled)
5182                         continue;
5183
5184                 config->sprites_enabled |= wm->sprites_enabled;
5185                 config->sprites_scaled |= wm->sprites_scaled;
5186                 config->num_pipes_active++;
5187         }
5188 }
5189
5190 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
5191 {
5192         struct drm_device *dev = &dev_priv->drm;
5193         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
5194         struct ilk_wm_maximums max;
5195         struct intel_wm_config config = {};
5196         struct ilk_wm_values results = {};
5197         enum intel_ddb_partitioning partitioning;
5198
5199         ilk_compute_wm_config(dev, &config);
5200
5201         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
5202         ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
5203
5204         /* 5/6 split only in single pipe config on IVB+ */
5205         if (INTEL_GEN(dev_priv) >= 7 &&
5206             config.num_pipes_active == 1 && config.sprites_enabled) {
5207                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
5208                 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
5209
5210                 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
5211         } else {
5212                 best_lp_wm = &lp_wm_1_2;
5213         }
5214
5215         partitioning = (best_lp_wm == &lp_wm_1_2) ?
5216                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
5217
5218         ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
5219
5220         ilk_write_wm_values(dev_priv, &results);
5221 }
5222
5223 static void ilk_initial_watermarks(struct intel_atomic_state *state,
5224                                    struct intel_crtc_state *cstate)
5225 {
5226         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5227         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5228
5229         mutex_lock(&dev_priv->wm.wm_mutex);
5230         intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
5231         ilk_program_watermarks(dev_priv);
5232         mutex_unlock(&dev_priv->wm.wm_mutex);
5233 }
5234
5235 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
5236                                     struct intel_crtc_state *cstate)
5237 {
5238         struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
5239         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
5240
5241         mutex_lock(&dev_priv->wm.wm_mutex);
5242         if (cstate->wm.need_postvbl_update) {
5243                 intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
5244                 ilk_program_watermarks(dev_priv);
5245         }
5246         mutex_unlock(&dev_priv->wm.wm_mutex);
5247 }
5248
5249 static inline void skl_wm_level_from_reg_val(uint32_t val,
5250                                              struct skl_wm_level *level)
5251 {
5252         level->plane_en = val & PLANE_WM_EN;
5253         level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
5254         level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
5255                 PLANE_WM_LINES_MASK;
5256 }
5257
5258 void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc,
5259                               struct skl_pipe_wm *out)
5260 {
5261         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
5262         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5263         enum pipe pipe = intel_crtc->pipe;
5264         int level, max_level;
5265         enum plane_id plane_id;
5266         uint32_t val;
5267
5268         max_level = ilk_wm_max_level(dev_priv);
5269
5270         for_each_plane_id_on_crtc(intel_crtc, plane_id) {
5271                 struct skl_plane_wm *wm = &out->planes[plane_id];
5272
5273                 for (level = 0; level <= max_level; level++) {
5274                         if (plane_id != PLANE_CURSOR)
5275                                 val = I915_READ(PLANE_WM(pipe, plane_id, level));
5276                         else
5277                                 val = I915_READ(CUR_WM(pipe, level));
5278
5279                         skl_wm_level_from_reg_val(val, &wm->wm[level]);
5280                 }
5281
5282                 if (plane_id != PLANE_CURSOR)
5283                         val = I915_READ(PLANE_WM_TRANS(pipe, plane_id));
5284                 else
5285                         val = I915_READ(CUR_WM_TRANS(pipe));
5286
5287                 skl_wm_level_from_reg_val(val, &wm->trans_wm);
5288         }
5289
5290         if (!intel_crtc->active)
5291                 return;
5292
5293         out->linetime = I915_READ(PIPE_WM_LINETIME(pipe));
5294 }
5295
5296 void skl_wm_get_hw_state(struct drm_device *dev)
5297 {
5298         struct drm_i915_private *dev_priv = to_i915(dev);
5299         struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
5300         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
5301         struct drm_crtc *crtc;
5302         struct intel_crtc *intel_crtc;
5303         struct intel_crtc_state *cstate;
5304
5305         skl_ddb_get_hw_state(dev_priv, ddb);
5306         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
5307                 intel_crtc = to_intel_crtc(crtc);
5308                 cstate = to_intel_crtc_state(crtc->state);
5309
5310                 skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal);
5311
5312                 if (intel_crtc->active)
5313                         hw->dirty_pipes |= drm_crtc_mask(crtc);
5314         }
5315
5316         if (dev_priv->active_crtcs) {
5317                 /* Fully recompute DDB on first atomic commit */
5318                 dev_priv->wm.distrust_bios_wm = true;
5319         } else {
5320                 /* Easy/common case; just sanitize DDB now if everything off */
5321                 memset(ddb, 0, sizeof(*ddb));
5322         }
5323 }
5324
5325 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
5326 {
5327         struct drm_device *dev = crtc->dev;
5328         struct drm_i915_private *dev_priv = to_i915(dev);
5329         struct ilk_wm_values *hw = &dev_priv->wm.hw;
5330         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5331         struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
5332         struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
5333         enum pipe pipe = intel_crtc->pipe;
5334         static const i915_reg_t wm0_pipe_reg[] = {
5335                 [PIPE_A] = WM0_PIPEA_ILK,
5336                 [PIPE_B] = WM0_PIPEB_ILK,
5337                 [PIPE_C] = WM0_PIPEC_IVB,
5338         };
5339
5340         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
5341         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5342                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5343
5344         memset(active, 0, sizeof(*active));
5345
5346         active->pipe_enabled = intel_crtc->active;
5347
5348         if (active->pipe_enabled) {
5349                 u32 tmp = hw->wm_pipe[pipe];
5350
5351                 /*
5352                  * For active pipes LP0 watermark is marked as
5353                  * enabled, and LP1+ watermaks as disabled since
5354                  * we can't really reverse compute them in case
5355                  * multiple pipes are active.
5356                  */
5357                 active->wm[0].enable = true;
5358                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
5359                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
5360                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
5361                 active->linetime = hw->wm_linetime[pipe];
5362         } else {
5363                 int level, max_level = ilk_wm_max_level(dev_priv);
5364
5365                 /*
5366                  * For inactive pipes, all watermark levels
5367                  * should be marked as enabled but zeroed,
5368                  * which is what we'd compute them to.
5369                  */
5370                 for (level = 0; level <= max_level; level++)
5371                         active->wm[level].enable = true;
5372         }
5373
5374         intel_crtc->wm.active.ilk = *active;
5375 }
5376
5377 #define _FW_WM(value, plane) \
5378         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
5379 #define _FW_WM_VLV(value, plane) \
5380         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
5381
5382 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
5383                                struct g4x_wm_values *wm)
5384 {
5385         uint32_t tmp;
5386
5387         tmp = I915_READ(DSPFW1);
5388         wm->sr.plane = _FW_WM(tmp, SR);
5389         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5390         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
5391         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
5392
5393         tmp = I915_READ(DSPFW2);
5394         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
5395         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
5396         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
5397         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
5398         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5399         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
5400
5401         tmp = I915_READ(DSPFW3);
5402         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
5403         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5404         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
5405         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
5406 }
5407
5408 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
5409                                struct vlv_wm_values *wm)
5410 {
5411         enum pipe pipe;
5412         uint32_t tmp;
5413
5414         for_each_pipe(dev_priv, pipe) {
5415                 tmp = I915_READ(VLV_DDL(pipe));
5416
5417                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
5418                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5419                 wm->ddl[pipe].plane[PLANE_CURSOR] =
5420                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5421                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
5422                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5423                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
5424                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
5425         }
5426
5427         tmp = I915_READ(DSPFW1);
5428         wm->sr.plane = _FW_WM(tmp, SR);
5429         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
5430         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
5431         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
5432
5433         tmp = I915_READ(DSPFW2);
5434         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
5435         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
5436         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
5437
5438         tmp = I915_READ(DSPFW3);
5439         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
5440
5441         if (IS_CHERRYVIEW(dev_priv)) {
5442                 tmp = I915_READ(DSPFW7_CHV);
5443                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5444                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5445
5446                 tmp = I915_READ(DSPFW8_CHV);
5447                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
5448                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
5449
5450                 tmp = I915_READ(DSPFW9_CHV);
5451                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
5452                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
5453
5454                 tmp = I915_READ(DSPHOWM);
5455                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5456                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
5457                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
5458                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
5459                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5460                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5461                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5462                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5463                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5464                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5465         } else {
5466                 tmp = I915_READ(DSPFW7);
5467                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
5468                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
5469
5470                 tmp = I915_READ(DSPHOWM);
5471                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
5472                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
5473                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
5474                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
5475                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
5476                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
5477                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
5478         }
5479 }
5480
5481 #undef _FW_WM
5482 #undef _FW_WM_VLV
5483
5484 void g4x_wm_get_hw_state(struct drm_device *dev)
5485 {
5486         struct drm_i915_private *dev_priv = to_i915(dev);
5487         struct g4x_wm_values *wm = &dev_priv->wm.g4x;
5488         struct intel_crtc *crtc;
5489
5490         g4x_read_wm_values(dev_priv, wm);
5491
5492         wm->cxsr = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN;
5493
5494         for_each_intel_crtc(dev, crtc) {
5495                 struct intel_crtc_state *crtc_state =
5496                         to_intel_crtc_state(crtc->base.state);
5497                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
5498                 struct g4x_pipe_wm *raw;
5499                 enum pipe pipe = crtc->pipe;
5500                 enum plane_id plane_id;
5501                 int level, max_level;
5502
5503                 active->cxsr = wm->cxsr;
5504                 active->hpll_en = wm->hpll_en;
5505                 active->fbc_en = wm->fbc_en;
5506
5507                 active->sr = wm->sr;
5508                 active->hpll = wm->hpll;
5509
5510                 for_each_plane_id_on_crtc(crtc, plane_id) {
5511                         active->wm.plane[plane_id] =
5512                                 wm->pipe[pipe].plane[plane_id];
5513                 }
5514
5515                 if (wm->cxsr && wm->hpll_en)
5516                         max_level = G4X_WM_LEVEL_HPLL;
5517                 else if (wm->cxsr)
5518                         max_level = G4X_WM_LEVEL_SR;
5519                 else
5520                         max_level = G4X_WM_LEVEL_NORMAL;
5521
5522                 level = G4X_WM_LEVEL_NORMAL;
5523                 raw = &crtc_state->wm.g4x.raw[level];
5524                 for_each_plane_id_on_crtc(crtc, plane_id)
5525                         raw->plane[plane_id] = active->wm.plane[plane_id];
5526
5527                 if (++level > max_level)
5528                         goto out;
5529
5530                 raw = &crtc_state->wm.g4x.raw[level];
5531                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
5532                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
5533                 raw->plane[PLANE_SPRITE0] = 0;
5534                 raw->fbc = active->sr.fbc;
5535
5536                 if (++level > max_level)
5537                         goto out;
5538
5539                 raw = &crtc_state->wm.g4x.raw[level];
5540                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
5541                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
5542                 raw->plane[PLANE_SPRITE0] = 0;
5543                 raw->fbc = active->hpll.fbc;
5544
5545         out:
5546                 for_each_plane_id_on_crtc(crtc, plane_id)
5547                         g4x_raw_plane_wm_set(crtc_state, level,
5548                                              plane_id, USHRT_MAX);
5549                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
5550
5551                 crtc_state->wm.g4x.optimal = *active;
5552                 crtc_state->wm.g4x.intermediate = *active;
5553
5554                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
5555                               pipe_name(pipe),
5556                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5557                               wm->pipe[pipe].plane[PLANE_CURSOR],
5558                               wm->pipe[pipe].plane[PLANE_SPRITE0]);
5559         }
5560
5561         DRM_DEBUG_KMS("Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
5562                       wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
5563         DRM_DEBUG_KMS("Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
5564                       wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
5565         DRM_DEBUG_KMS("Initial SR=%s HPLL=%s FBC=%s\n",
5566                       yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
5567 }
5568
5569 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
5570 {
5571         struct intel_plane *plane;
5572         struct intel_crtc *crtc;
5573
5574         mutex_lock(&dev_priv->wm.wm_mutex);
5575
5576         for_each_intel_plane(&dev_priv->drm, plane) {
5577                 struct intel_crtc *crtc =
5578                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5579                 struct intel_crtc_state *crtc_state =
5580                         to_intel_crtc_state(crtc->base.state);
5581                 struct intel_plane_state *plane_state =
5582                         to_intel_plane_state(plane->base.state);
5583                 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
5584                 enum plane_id plane_id = plane->id;
5585                 int level;
5586
5587                 if (plane_state->base.visible)
5588                         continue;
5589
5590                 for (level = 0; level < 3; level++) {
5591                         struct g4x_pipe_wm *raw =
5592                                 &crtc_state->wm.g4x.raw[level];
5593
5594                         raw->plane[plane_id] = 0;
5595                         wm_state->wm.plane[plane_id] = 0;
5596                 }
5597
5598                 if (plane_id == PLANE_PRIMARY) {
5599                         for (level = 0; level < 3; level++) {
5600                                 struct g4x_pipe_wm *raw =
5601                                         &crtc_state->wm.g4x.raw[level];
5602                                 raw->fbc = 0;
5603                         }
5604
5605                         wm_state->sr.fbc = 0;
5606                         wm_state->hpll.fbc = 0;
5607                         wm_state->fbc_en = false;
5608                 }
5609         }
5610
5611         for_each_intel_crtc(&dev_priv->drm, crtc) {
5612                 struct intel_crtc_state *crtc_state =
5613                         to_intel_crtc_state(crtc->base.state);
5614
5615                 crtc_state->wm.g4x.intermediate =
5616                         crtc_state->wm.g4x.optimal;
5617                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
5618         }
5619
5620         g4x_program_watermarks(dev_priv);
5621
5622         mutex_unlock(&dev_priv->wm.wm_mutex);
5623 }
5624
5625 void vlv_wm_get_hw_state(struct drm_device *dev)
5626 {
5627         struct drm_i915_private *dev_priv = to_i915(dev);
5628         struct vlv_wm_values *wm = &dev_priv->wm.vlv;
5629         struct intel_crtc *crtc;
5630         u32 val;
5631
5632         vlv_read_wm_values(dev_priv, wm);
5633
5634         wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
5635         wm->level = VLV_WM_LEVEL_PM2;
5636
5637         if (IS_CHERRYVIEW(dev_priv)) {
5638                 mutex_lock(&dev_priv->pcu_lock);
5639
5640                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5641                 if (val & DSP_MAXFIFO_PM5_ENABLE)
5642                         wm->level = VLV_WM_LEVEL_PM5;
5643
5644                 /*
5645                  * If DDR DVFS is disabled in the BIOS, Punit
5646                  * will never ack the request. So if that happens
5647                  * assume we don't have to enable/disable DDR DVFS
5648                  * dynamically. To test that just set the REQ_ACK
5649                  * bit to poke the Punit, but don't change the
5650                  * HIGH/LOW bits so that we don't actually change
5651                  * the current state.
5652                  */
5653                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5654                 val |= FORCE_DDR_FREQ_REQ_ACK;
5655                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
5656
5657                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
5658                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
5659                         DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
5660                                       "assuming DDR DVFS is disabled\n");
5661                         dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
5662                 } else {
5663                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
5664                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
5665                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
5666                 }
5667
5668                 mutex_unlock(&dev_priv->pcu_lock);
5669         }
5670
5671         for_each_intel_crtc(dev, crtc) {
5672                 struct intel_crtc_state *crtc_state =
5673                         to_intel_crtc_state(crtc->base.state);
5674                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
5675                 const struct vlv_fifo_state *fifo_state =
5676                         &crtc_state->wm.vlv.fifo_state;
5677                 enum pipe pipe = crtc->pipe;
5678                 enum plane_id plane_id;
5679                 int level;
5680
5681                 vlv_get_fifo_size(crtc_state);
5682
5683                 active->num_levels = wm->level + 1;
5684                 active->cxsr = wm->cxsr;
5685
5686                 for (level = 0; level < active->num_levels; level++) {
5687                         struct g4x_pipe_wm *raw =
5688                                 &crtc_state->wm.vlv.raw[level];
5689
5690                         active->sr[level].plane = wm->sr.plane;
5691                         active->sr[level].cursor = wm->sr.cursor;
5692
5693                         for_each_plane_id_on_crtc(crtc, plane_id) {
5694                                 active->wm[level].plane[plane_id] =
5695                                         wm->pipe[pipe].plane[plane_id];
5696
5697                                 raw->plane[plane_id] =
5698                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
5699                                                             fifo_state->plane[plane_id]);
5700                         }
5701                 }
5702
5703                 for_each_plane_id_on_crtc(crtc, plane_id)
5704                         vlv_raw_plane_wm_set(crtc_state, level,
5705                                              plane_id, USHRT_MAX);
5706                 vlv_invalidate_wms(crtc, active, level);
5707
5708                 crtc_state->wm.vlv.optimal = *active;
5709                 crtc_state->wm.vlv.intermediate = *active;
5710
5711                 DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
5712                               pipe_name(pipe),
5713                               wm->pipe[pipe].plane[PLANE_PRIMARY],
5714                               wm->pipe[pipe].plane[PLANE_CURSOR],
5715                               wm->pipe[pipe].plane[PLANE_SPRITE0],
5716                               wm->pipe[pipe].plane[PLANE_SPRITE1]);
5717         }
5718
5719         DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
5720                       wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
5721 }
5722
5723 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
5724 {
5725         struct intel_plane *plane;
5726         struct intel_crtc *crtc;
5727
5728         mutex_lock(&dev_priv->wm.wm_mutex);
5729
5730         for_each_intel_plane(&dev_priv->drm, plane) {
5731                 struct intel_crtc *crtc =
5732                         intel_get_crtc_for_pipe(dev_priv, plane->pipe);
5733                 struct intel_crtc_state *crtc_state =
5734                         to_intel_crtc_state(crtc->base.state);
5735                 struct intel_plane_state *plane_state =
5736                         to_intel_plane_state(plane->base.state);
5737                 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
5738                 const struct vlv_fifo_state *fifo_state =
5739                         &crtc_state->wm.vlv.fifo_state;
5740                 enum plane_id plane_id = plane->id;
5741                 int level;
5742
5743                 if (plane_state->base.visible)
5744                         continue;
5745
5746                 for (level = 0; level < wm_state->num_levels; level++) {
5747                         struct g4x_pipe_wm *raw =
5748                                 &crtc_state->wm.vlv.raw[level];
5749
5750                         raw->plane[plane_id] = 0;
5751
5752                         wm_state->wm[level].plane[plane_id] =
5753                                 vlv_invert_wm_value(raw->plane[plane_id],
5754                                                     fifo_state->plane[plane_id]);
5755                 }
5756         }
5757
5758         for_each_intel_crtc(&dev_priv->drm, crtc) {
5759                 struct intel_crtc_state *crtc_state =
5760                         to_intel_crtc_state(crtc->base.state);
5761
5762                 crtc_state->wm.vlv.intermediate =
5763                         crtc_state->wm.vlv.optimal;
5764                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
5765         }
5766
5767         vlv_program_watermarks(dev_priv);
5768
5769         mutex_unlock(&dev_priv->wm.wm_mutex);
5770 }
5771
5772 /*
5773  * FIXME should probably kill this and improve
5774  * the real watermark readout/sanitation instead
5775  */
5776 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
5777 {
5778         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
5779         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
5780         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
5781
5782         /*
5783          * Don't touch WM1S_LP_EN here.
5784          * Doing so could cause underruns.
5785          */
5786 }
5787
5788 void ilk_wm_get_hw_state(struct drm_device *dev)
5789 {
5790         struct drm_i915_private *dev_priv = to_i915(dev);
5791         struct ilk_wm_values *hw = &dev_priv->wm.hw;
5792         struct drm_crtc *crtc;
5793
5794         ilk_init_lp_watermarks(dev_priv);
5795
5796         for_each_crtc(dev, crtc)
5797                 ilk_pipe_wm_get_hw_state(crtc);
5798
5799         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
5800         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
5801         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
5802
5803         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5804         if (INTEL_GEN(dev_priv) >= 7) {
5805                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
5806                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5807         }
5808
5809         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5810                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
5811                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
5812         else if (IS_IVYBRIDGE(dev_priv))
5813                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
5814                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
5815
5816         hw->enable_fbc_wm =
5817                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
5818 }
5819
5820 /**
5821  * intel_update_watermarks - update FIFO watermark values based on current modes
5822  *
5823  * Calculate watermark values for the various WM regs based on current mode
5824  * and plane configuration.
5825  *
5826  * There are several cases to deal with here:
5827  *   - normal (i.e. non-self-refresh)
5828  *   - self-refresh (SR) mode
5829  *   - lines are large relative to FIFO size (buffer can hold up to 2)
5830  *   - lines are small relative to FIFO size (buffer can hold more than 2
5831  *     lines), so need to account for TLB latency
5832  *
5833  *   The normal calculation is:
5834  *     watermark = dotclock * bytes per pixel * latency
5835  *   where latency is platform & configuration dependent (we assume pessimal
5836  *   values here).
5837  *
5838  *   The SR calculation is:
5839  *     watermark = (trunc(latency/line time)+1) * surface width *
5840  *       bytes per pixel
5841  *   where
5842  *     line time = htotal / dotclock
5843  *     surface width = hdisplay for normal plane and 64 for cursor
5844  *   and latency is assumed to be high, as above.
5845  *
5846  * The final value programmed to the register should always be rounded up,
5847  * and include an extra 2 entries to account for clock crossings.
5848  *
5849  * We don't use the sprite, so we can ignore that.  And on Crestline we have
5850  * to set the non-SR watermarks to 8.
5851  */
5852 void intel_update_watermarks(struct intel_crtc *crtc)
5853 {
5854         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5855
5856         if (dev_priv->display.update_wm)
5857                 dev_priv->display.update_wm(crtc);
5858 }
5859
5860 void intel_enable_ipc(struct drm_i915_private *dev_priv)
5861 {
5862         u32 val;
5863
5864         /* Display WA #0477 WaDisableIPC: skl */
5865         if (IS_SKYLAKE(dev_priv)) {
5866                 dev_priv->ipc_enabled = false;
5867                 return;
5868         }
5869
5870         val = I915_READ(DISP_ARB_CTL2);
5871
5872         if (dev_priv->ipc_enabled)
5873                 val |= DISP_IPC_ENABLE;
5874         else
5875                 val &= ~DISP_IPC_ENABLE;
5876
5877         I915_WRITE(DISP_ARB_CTL2, val);
5878 }
5879
5880 void intel_init_ipc(struct drm_i915_private *dev_priv)
5881 {
5882         dev_priv->ipc_enabled = false;
5883         if (!HAS_IPC(dev_priv))
5884                 return;
5885
5886         dev_priv->ipc_enabled = true;
5887         intel_enable_ipc(dev_priv);
5888 }
5889
5890 /*
5891  * Lock protecting IPS related data structures
5892  */
5893 DEFINE_SPINLOCK(mchdev_lock);
5894
5895 /* Global for IPS driver to get at the current i915 device. Protected by
5896  * mchdev_lock. */
5897 static struct drm_i915_private *i915_mch_dev;
5898
5899 bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
5900 {
5901         u16 rgvswctl;
5902
5903         lockdep_assert_held(&mchdev_lock);
5904
5905         rgvswctl = I915_READ16(MEMSWCTL);
5906         if (rgvswctl & MEMCTL_CMD_STS) {
5907                 DRM_DEBUG("gpu busy, RCS change rejected\n");
5908                 return false; /* still busy with another command */
5909         }
5910
5911         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
5912                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
5913         I915_WRITE16(MEMSWCTL, rgvswctl);
5914         POSTING_READ16(MEMSWCTL);
5915
5916         rgvswctl |= MEMCTL_CMD_STS;
5917         I915_WRITE16(MEMSWCTL, rgvswctl);
5918
5919         return true;
5920 }
5921
5922 static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
5923 {
5924         u32 rgvmodectl;
5925         u8 fmax, fmin, fstart, vstart;
5926
5927         spin_lock_irq(&mchdev_lock);
5928
5929         rgvmodectl = I915_READ(MEMMODECTL);
5930
5931         /* Enable temp reporting */
5932         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
5933         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
5934
5935         /* 100ms RC evaluation intervals */
5936         I915_WRITE(RCUPEI, 100000);
5937         I915_WRITE(RCDNEI, 100000);
5938
5939         /* Set max/min thresholds to 90ms and 80ms respectively */
5940         I915_WRITE(RCBMAXAVG, 90000);
5941         I915_WRITE(RCBMINAVG, 80000);
5942
5943         I915_WRITE(MEMIHYST, 1);
5944
5945         /* Set up min, max, and cur for interrupt handling */
5946         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
5947         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
5948         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
5949                 MEMMODE_FSTART_SHIFT;
5950
5951         vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
5952                 PXVFREQ_PX_SHIFT;
5953
5954         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
5955         dev_priv->ips.fstart = fstart;
5956
5957         dev_priv->ips.max_delay = fstart;
5958         dev_priv->ips.min_delay = fmin;
5959         dev_priv->ips.cur_delay = fstart;
5960
5961         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
5962                          fmax, fmin, fstart);
5963
5964         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
5965
5966         /*
5967          * Interrupts will be enabled in ironlake_irq_postinstall
5968          */
5969
5970         I915_WRITE(VIDSTART, vstart);
5971         POSTING_READ(VIDSTART);
5972
5973         rgvmodectl |= MEMMODE_SWMODE_EN;
5974         I915_WRITE(MEMMODECTL, rgvmodectl);
5975
5976         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
5977                 DRM_ERROR("stuck trying to change perf mode\n");
5978         mdelay(1);
5979
5980         ironlake_set_drps(dev_priv, fstart);
5981
5982         dev_priv->ips.last_count1 = I915_READ(DMIEC) +
5983                 I915_READ(DDREC) + I915_READ(CSIEC);
5984         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
5985         dev_priv->ips.last_count2 = I915_READ(GFXEC);
5986         dev_priv->ips.last_time2 = ktime_get_raw_ns();
5987
5988         spin_unlock_irq(&mchdev_lock);
5989 }
5990
5991 static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
5992 {
5993         u16 rgvswctl;
5994
5995         spin_lock_irq(&mchdev_lock);
5996
5997         rgvswctl = I915_READ16(MEMSWCTL);
5998
5999         /* Ack interrupts, disable EFC interrupt */
6000         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
6001         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
6002         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
6003         I915_WRITE(DEIIR, DE_PCU_EVENT);
6004         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
6005
6006         /* Go back to the starting frequency */
6007         ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
6008         mdelay(1);
6009         rgvswctl |= MEMCTL_CMD_STS;
6010         I915_WRITE(MEMSWCTL, rgvswctl);
6011         mdelay(1);
6012
6013         spin_unlock_irq(&mchdev_lock);
6014 }
6015
6016 /* There's a funny hw issue where the hw returns all 0 when reading from
6017  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
6018  * ourselves, instead of doing a rmw cycle (which might result in us clearing
6019  * all limits and the gpu stuck at whatever frequency it is at atm).
6020  */
6021 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
6022 {
6023         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6024         u32 limits;
6025
6026         /* Only set the down limit when we've reached the lowest level to avoid
6027          * getting more interrupts, otherwise leave this clear. This prevents a
6028          * race in the hw when coming out of rc6: There's a tiny window where
6029          * the hw runs at the minimal clock before selecting the desired
6030          * frequency, if the down threshold expires in that window we will not
6031          * receive a down interrupt. */
6032         if (INTEL_GEN(dev_priv) >= 9) {
6033                 limits = (rps->max_freq_softlimit) << 23;
6034                 if (val <= rps->min_freq_softlimit)
6035                         limits |= (rps->min_freq_softlimit) << 14;
6036         } else {
6037                 limits = rps->max_freq_softlimit << 24;
6038                 if (val <= rps->min_freq_softlimit)
6039                         limits |= rps->min_freq_softlimit << 16;
6040         }
6041
6042         return limits;
6043 }
6044
6045 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
6046 {
6047         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6048         int new_power;
6049         u32 threshold_up = 0, threshold_down = 0; /* in % */
6050         u32 ei_up = 0, ei_down = 0;
6051
6052         new_power = rps->power;
6053         switch (rps->power) {
6054         case LOW_POWER:
6055                 if (val > rps->efficient_freq + 1 &&
6056                     val > rps->cur_freq)
6057                         new_power = BETWEEN;
6058                 break;
6059
6060         case BETWEEN:
6061                 if (val <= rps->efficient_freq &&
6062                     val < rps->cur_freq)
6063                         new_power = LOW_POWER;
6064                 else if (val >= rps->rp0_freq &&
6065                          val > rps->cur_freq)
6066                         new_power = HIGH_POWER;
6067                 break;
6068
6069         case HIGH_POWER:
6070                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
6071                     val < rps->cur_freq)
6072                         new_power = BETWEEN;
6073                 break;
6074         }
6075         /* Max/min bins are special */
6076         if (val <= rps->min_freq_softlimit)
6077                 new_power = LOW_POWER;
6078         if (val >= rps->max_freq_softlimit)
6079                 new_power = HIGH_POWER;
6080         if (new_power == rps->power)
6081                 return;
6082
6083         /* Note the units here are not exactly 1us, but 1280ns. */
6084         switch (new_power) {
6085         case LOW_POWER:
6086                 /* Upclock if more than 95% busy over 16ms */
6087                 ei_up = 16000;
6088                 threshold_up = 95;
6089
6090                 /* Downclock if less than 85% busy over 32ms */
6091                 ei_down = 32000;
6092                 threshold_down = 85;
6093                 break;
6094
6095         case BETWEEN:
6096                 /* Upclock if more than 90% busy over 13ms */
6097                 ei_up = 13000;
6098                 threshold_up = 90;
6099
6100                 /* Downclock if less than 75% busy over 32ms */
6101                 ei_down = 32000;
6102                 threshold_down = 75;
6103                 break;
6104
6105         case HIGH_POWER:
6106                 /* Upclock if more than 85% busy over 10ms */
6107                 ei_up = 10000;
6108                 threshold_up = 85;
6109
6110                 /* Downclock if less than 60% busy over 32ms */
6111                 ei_down = 32000;
6112                 threshold_down = 60;
6113                 break;
6114         }
6115
6116         /* When byt can survive without system hang with dynamic
6117          * sw freq adjustments, this restriction can be lifted.
6118          */
6119         if (IS_VALLEYVIEW(dev_priv))
6120                 goto skip_hw_write;
6121
6122         I915_WRITE(GEN6_RP_UP_EI,
6123                    GT_INTERVAL_FROM_US(dev_priv, ei_up));
6124         I915_WRITE(GEN6_RP_UP_THRESHOLD,
6125                    GT_INTERVAL_FROM_US(dev_priv,
6126                                        ei_up * threshold_up / 100));
6127
6128         I915_WRITE(GEN6_RP_DOWN_EI,
6129                    GT_INTERVAL_FROM_US(dev_priv, ei_down));
6130         I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
6131                    GT_INTERVAL_FROM_US(dev_priv,
6132                                        ei_down * threshold_down / 100));
6133
6134         I915_WRITE(GEN6_RP_CONTROL,
6135                    GEN6_RP_MEDIA_TURBO |
6136                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
6137                    GEN6_RP_MEDIA_IS_GFX |
6138                    GEN6_RP_ENABLE |
6139                    GEN6_RP_UP_BUSY_AVG |
6140                    GEN6_RP_DOWN_IDLE_AVG);
6141
6142 skip_hw_write:
6143         rps->power = new_power;
6144         rps->up_threshold = threshold_up;
6145         rps->down_threshold = threshold_down;
6146         rps->last_adj = 0;
6147 }
6148
6149 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
6150 {
6151         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6152         u32 mask = 0;
6153
6154         /* We use UP_EI_EXPIRED interupts for both up/down in manual mode */
6155         if (val > rps->min_freq_softlimit)
6156                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
6157         if (val < rps->max_freq_softlimit)
6158                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
6159
6160         mask &= dev_priv->pm_rps_events;
6161
6162         return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
6163 }
6164
6165 /* gen6_set_rps is called to update the frequency request, but should also be
6166  * called when the range (min_delay and max_delay) is modified so that we can
6167  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6168 static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
6169 {
6170         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6171
6172         /* min/max delay may still have been modified so be sure to
6173          * write the limits value.
6174          */
6175         if (val != rps->cur_freq) {
6176                 gen6_set_rps_thresholds(dev_priv, val);
6177
6178                 if (INTEL_GEN(dev_priv) >= 9)
6179                         I915_WRITE(GEN6_RPNSWREQ,
6180                                    GEN9_FREQUENCY(val));
6181                 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
6182                         I915_WRITE(GEN6_RPNSWREQ,
6183                                    HSW_FREQUENCY(val));
6184                 else
6185                         I915_WRITE(GEN6_RPNSWREQ,
6186                                    GEN6_FREQUENCY(val) |
6187                                    GEN6_OFFSET(0) |
6188                                    GEN6_AGGRESSIVE_TURBO);
6189         }
6190
6191         /* Make sure we continue to get interrupts
6192          * until we hit the minimum or maximum frequencies.
6193          */
6194         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
6195         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6196
6197         rps->cur_freq = val;
6198         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6199
6200         return 0;
6201 }
6202
6203 static int valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
6204 {
6205         int err;
6206
6207         if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
6208                       "Odd GPU freq value\n"))
6209                 val &= ~1;
6210
6211         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
6212
6213         if (val != dev_priv->gt_pm.rps.cur_freq) {
6214                 err = vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
6215                 if (err)
6216                         return err;
6217
6218                 gen6_set_rps_thresholds(dev_priv, val);
6219         }
6220
6221         dev_priv->gt_pm.rps.cur_freq = val;
6222         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
6223
6224         return 0;
6225 }
6226
6227 /* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
6228  *
6229  * * If Gfx is Idle, then
6230  * 1. Forcewake Media well.
6231  * 2. Request idle freq.
6232  * 3. Release Forcewake of Media well.
6233 */
6234 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
6235 {
6236         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6237         u32 val = rps->idle_freq;
6238         int err;
6239
6240         if (rps->cur_freq <= val)
6241                 return;
6242
6243         /* The punit delays the write of the frequency and voltage until it
6244          * determines the GPU is awake. During normal usage we don't want to
6245          * waste power changing the frequency if the GPU is sleeping (rc6).
6246          * However, the GPU and driver is now idle and we do not want to delay
6247          * switching to minimum voltage (reducing power whilst idle) as we do
6248          * not expect to be woken in the near future and so must flush the
6249          * change by waking the device.
6250          *
6251          * We choose to take the media powerwell (either would do to trick the
6252          * punit into committing the voltage change) as that takes a lot less
6253          * power than the render powerwell.
6254          */
6255         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
6256         err = valleyview_set_rps(dev_priv, val);
6257         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
6258
6259         if (err)
6260                 DRM_ERROR("Failed to set RPS for idle\n");
6261 }
6262
6263 void gen6_rps_busy(struct drm_i915_private *dev_priv)
6264 {
6265         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6266
6267         mutex_lock(&dev_priv->pcu_lock);
6268         if (rps->enabled) {
6269                 u8 freq;
6270
6271                 if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6272                         gen6_rps_reset_ei(dev_priv);
6273                 I915_WRITE(GEN6_PMINTRMSK,
6274                            gen6_rps_pm_mask(dev_priv, rps->cur_freq));
6275
6276                 gen6_enable_rps_interrupts(dev_priv);
6277
6278                 /* Use the user's desired frequency as a guide, but for better
6279                  * performance, jump directly to RPe as our starting frequency.
6280                  */
6281                 freq = max(rps->cur_freq,
6282                            rps->efficient_freq);
6283
6284                 if (intel_set_rps(dev_priv,
6285                                   clamp(freq,
6286                                         rps->min_freq_softlimit,
6287                                         rps->max_freq_softlimit)))
6288                         DRM_DEBUG_DRIVER("Failed to set idle frequency\n");
6289         }
6290         mutex_unlock(&dev_priv->pcu_lock);
6291 }
6292
6293 void gen6_rps_idle(struct drm_i915_private *dev_priv)
6294 {
6295         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6296
6297         /* Flush our bottom-half so that it does not race with us
6298          * setting the idle frequency and so that it is bounded by
6299          * our rpm wakeref. And then disable the interrupts to stop any
6300          * futher RPS reclocking whilst we are asleep.
6301          */
6302         gen6_disable_rps_interrupts(dev_priv);
6303
6304         mutex_lock(&dev_priv->pcu_lock);
6305         if (rps->enabled) {
6306                 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6307                         vlv_set_rps_idle(dev_priv);
6308                 else
6309                         gen6_set_rps(dev_priv, rps->idle_freq);
6310                 rps->last_adj = 0;
6311                 I915_WRITE(GEN6_PMINTRMSK,
6312                            gen6_sanitize_rps_pm_mask(dev_priv, ~0));
6313         }
6314         mutex_unlock(&dev_priv->pcu_lock);
6315 }
6316
6317 void gen6_rps_boost(struct drm_i915_gem_request *rq,
6318                     struct intel_rps_client *rps_client)
6319 {
6320         struct intel_rps *rps = &rq->i915->gt_pm.rps;
6321         unsigned long flags;
6322         bool boost;
6323
6324         /* This is intentionally racy! We peek at the state here, then
6325          * validate inside the RPS worker.
6326          */
6327         if (!rps->enabled)
6328                 return;
6329
6330         boost = false;
6331         spin_lock_irqsave(&rq->lock, flags);
6332         if (!rq->waitboost && !i915_gem_request_completed(rq)) {
6333                 atomic_inc(&rps->num_waiters);
6334                 rq->waitboost = true;
6335                 boost = true;
6336         }
6337         spin_unlock_irqrestore(&rq->lock, flags);
6338         if (!boost)
6339                 return;
6340
6341         if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
6342                 schedule_work(&rps->work);
6343
6344         atomic_inc(rps_client ? &rps_client->boosts : &rps->boosts);
6345 }
6346
6347 int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
6348 {
6349         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6350         int err;
6351
6352         lockdep_assert_held(&dev_priv->pcu_lock);
6353         GEM_BUG_ON(val > rps->max_freq);
6354         GEM_BUG_ON(val < rps->min_freq);
6355
6356         if (!rps->enabled) {
6357                 rps->cur_freq = val;
6358                 return 0;
6359         }
6360
6361         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
6362                 err = valleyview_set_rps(dev_priv, val);
6363         else
6364                 err = gen6_set_rps(dev_priv, val);
6365
6366         return err;
6367 }
6368
6369 static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
6370 {
6371         I915_WRITE(GEN6_RC_CONTROL, 0);
6372         I915_WRITE(GEN9_PG_ENABLE, 0);
6373 }
6374
6375 static void gen9_disable_rps(struct drm_i915_private *dev_priv)
6376 {
6377         I915_WRITE(GEN6_RP_CONTROL, 0);
6378 }
6379
6380 static void gen6_disable_rc6(struct drm_i915_private *dev_priv)
6381 {
6382         I915_WRITE(GEN6_RC_CONTROL, 0);
6383 }
6384
6385 static void gen6_disable_rps(struct drm_i915_private *dev_priv)
6386 {
6387         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
6388         I915_WRITE(GEN6_RP_CONTROL, 0);
6389 }
6390
6391 static void cherryview_disable_rc6(struct drm_i915_private *dev_priv)
6392 {
6393         I915_WRITE(GEN6_RC_CONTROL, 0);
6394 }
6395
6396 static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
6397 {
6398         I915_WRITE(GEN6_RP_CONTROL, 0);
6399 }
6400
6401 static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
6402 {
6403         /* We're doing forcewake before Disabling RC6,
6404          * This what the BIOS expects when going into suspend */
6405         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6406
6407         I915_WRITE(GEN6_RC_CONTROL, 0);
6408
6409         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6410 }
6411
6412 static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
6413 {
6414         I915_WRITE(GEN6_RP_CONTROL, 0);
6415 }
6416
6417 static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
6418 {
6419         bool enable_rc6 = true;
6420         unsigned long rc6_ctx_base;
6421         u32 rc_ctl;
6422         int rc_sw_target;
6423
6424         rc_ctl = I915_READ(GEN6_RC_CONTROL);
6425         rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
6426                        RC_SW_TARGET_STATE_SHIFT;
6427         DRM_DEBUG_DRIVER("BIOS enabled RC states: "
6428                          "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
6429                          onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
6430                          onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
6431                          rc_sw_target);
6432
6433         if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
6434                 DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
6435                 enable_rc6 = false;
6436         }
6437
6438         /*
6439          * The exact context size is not known for BXT, so assume a page size
6440          * for this check.
6441          */
6442         rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
6443         if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
6444               (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
6445                 DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
6446                 enable_rc6 = false;
6447         }
6448
6449         if (!(((I915_READ(PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1) &&
6450               ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
6451               ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
6452               ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
6453                 DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
6454                 enable_rc6 = false;
6455         }
6456
6457         if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
6458             !I915_READ(GEN8_PUSHBUS_ENABLE) ||
6459             !I915_READ(GEN8_PUSHBUS_SHIFT)) {
6460                 DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
6461                 enable_rc6 = false;
6462         }
6463
6464         if (!I915_READ(GEN6_GFXPAUSE)) {
6465                 DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
6466                 enable_rc6 = false;
6467         }
6468
6469         if (!I915_READ(GEN8_MISC_CTRL0)) {
6470                 DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
6471                 enable_rc6 = false;
6472         }
6473
6474         return enable_rc6;
6475 }
6476
6477 static bool sanitize_rc6(struct drm_i915_private *i915)
6478 {
6479         struct intel_device_info *info = mkwrite_device_info(i915);
6480
6481         /* Powersaving is controlled by the host when inside a VM */
6482         if (intel_vgpu_active(i915))
6483                 info->has_rc6 = 0;
6484
6485         if (info->has_rc6 &&
6486             IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
6487                 DRM_INFO("RC6 disabled by BIOS\n");
6488                 info->has_rc6 = 0;
6489         }
6490
6491         /*
6492          * We assume that we do not have any deep rc6 levels if we don't have
6493          * have the previous rc6 level supported, i.e. we use HAS_RC6()
6494          * as the initial coarse check for rc6 in general, moving on to
6495          * progressively finer/deeper levels.
6496          */
6497         if (!info->has_rc6 && info->has_rc6p)
6498                 info->has_rc6p = 0;
6499
6500         return info->has_rc6;
6501 }
6502
6503 static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
6504 {
6505         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6506
6507         /* All of these values are in units of 50MHz */
6508
6509         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
6510         if (IS_GEN9_LP(dev_priv)) {
6511                 u32 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
6512                 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
6513                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6514                 rps->min_freq = (rp_state_cap >>  0) & 0xff;
6515         } else {
6516                 u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
6517                 rps->rp0_freq = (rp_state_cap >>  0) & 0xff;
6518                 rps->rp1_freq = (rp_state_cap >>  8) & 0xff;
6519                 rps->min_freq = (rp_state_cap >> 16) & 0xff;
6520         }
6521         /* hw_max = RP0 until we check for overclocking */
6522         rps->max_freq = rps->rp0_freq;
6523
6524         rps->efficient_freq = rps->rp1_freq;
6525         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
6526             IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
6527                 u32 ddcc_status = 0;
6528
6529                 if (sandybridge_pcode_read(dev_priv,
6530                                            HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
6531                                            &ddcc_status) == 0)
6532                         rps->efficient_freq =
6533                                 clamp_t(u8,
6534                                         ((ddcc_status >> 8) & 0xff),
6535                                         rps->min_freq,
6536                                         rps->max_freq);
6537         }
6538
6539         if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
6540                 /* Store the frequency values in 16.66 MHZ units, which is
6541                  * the natural hardware unit for SKL
6542                  */
6543                 rps->rp0_freq *= GEN9_FREQ_SCALER;
6544                 rps->rp1_freq *= GEN9_FREQ_SCALER;
6545                 rps->min_freq *= GEN9_FREQ_SCALER;
6546                 rps->max_freq *= GEN9_FREQ_SCALER;
6547                 rps->efficient_freq *= GEN9_FREQ_SCALER;
6548         }
6549 }
6550
6551 static void reset_rps(struct drm_i915_private *dev_priv,
6552                       int (*set)(struct drm_i915_private *, u8))
6553 {
6554         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6555         u8 freq = rps->cur_freq;
6556
6557         /* force a reset */
6558         rps->power = -1;
6559         rps->cur_freq = -1;
6560
6561         if (set(dev_priv, freq))
6562                 DRM_ERROR("Failed to reset RPS to initial values\n");
6563 }
6564
6565 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
6566 static void gen9_enable_rps(struct drm_i915_private *dev_priv)
6567 {
6568         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6569
6570         /* Program defaults and thresholds for RPS */
6571         if (IS_GEN9(dev_priv))
6572                 I915_WRITE(GEN6_RC_VIDEO_FREQ,
6573                         GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
6574
6575         /* 1 second timeout*/
6576         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
6577                 GT_INTERVAL_FROM_US(dev_priv, 1000000));
6578
6579         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
6580
6581         /* Leaning on the below call to gen6_set_rps to program/setup the
6582          * Up/Down EI & threshold registers, as well as the RP_CONTROL,
6583          * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
6584         reset_rps(dev_priv, gen6_set_rps);
6585
6586         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6587 }
6588
6589 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
6590 {
6591         struct intel_engine_cs *engine;
6592         enum intel_engine_id id;
6593         u32 rc6_mode;
6594
6595         /* 1a: Software RC state - RC0 */
6596         I915_WRITE(GEN6_RC_STATE, 0);
6597
6598         /* 1b: Get forcewake during program sequence. Although the driver
6599          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6600         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6601
6602         /* 2a: Disable RC states. */
6603         I915_WRITE(GEN6_RC_CONTROL, 0);
6604
6605         /* 2b: Program RC6 thresholds.*/
6606         if (INTEL_GEN(dev_priv) >= 10) {
6607                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
6608                 I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
6609         } else if (IS_SKYLAKE(dev_priv)) {
6610                 /*
6611                  * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
6612                  * when CPG is enabled
6613                  */
6614                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
6615         } else {
6616                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
6617         }
6618
6619         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
6620         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
6621         for_each_engine(engine, dev_priv, id)
6622                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6623
6624         if (HAS_GUC(dev_priv))
6625                 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
6626
6627         I915_WRITE(GEN6_RC_SLEEP, 0);
6628
6629         /* 2c: Program Coarse Power Gating Policies. */
6630         I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
6631         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
6632
6633         /* 3a: Enable RC6 */
6634         I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
6635
6636         /* WaRsUseTimeoutMode:cnl (pre-prod) */
6637         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_C0))
6638                 rc6_mode = GEN7_RC_CTL_TO_MODE;
6639         else
6640                 rc6_mode = GEN6_RC_CTL_EI_MODE(1);
6641
6642         I915_WRITE(GEN6_RC_CONTROL,
6643                    GEN6_RC_CTL_HW_ENABLE |
6644                    GEN6_RC_CTL_RC6_ENABLE |
6645                    rc6_mode);
6646
6647         /*
6648          * 3b: Enable Coarse Power Gating only when RC6 is enabled.
6649          * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
6650          */
6651         if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
6652                 I915_WRITE(GEN9_PG_ENABLE, 0);
6653         else
6654                 I915_WRITE(GEN9_PG_ENABLE,
6655                            GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
6656
6657         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6658 }
6659
6660 static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
6661 {
6662         struct intel_engine_cs *engine;
6663         enum intel_engine_id id;
6664
6665         /* 1a: Software RC state - RC0 */
6666         I915_WRITE(GEN6_RC_STATE, 0);
6667
6668         /* 1b: Get forcewake during program sequence. Although the driver
6669          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6670         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6671
6672         /* 2a: Disable RC states. */
6673         I915_WRITE(GEN6_RC_CONTROL, 0);
6674
6675         /* 2b: Program RC6 thresholds.*/
6676         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
6677         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
6678         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
6679         for_each_engine(engine, dev_priv, id)
6680                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6681         I915_WRITE(GEN6_RC_SLEEP, 0);
6682         I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
6683
6684         /* 3: Enable RC6 */
6685
6686         I915_WRITE(GEN6_RC_CONTROL,
6687                    GEN6_RC_CTL_HW_ENABLE |
6688                    GEN7_RC_CTL_TO_MODE |
6689                    GEN6_RC_CTL_RC6_ENABLE);
6690
6691         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6692 }
6693
6694 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
6695 {
6696         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6697
6698         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6699
6700         /* 1 Program defaults and thresholds for RPS*/
6701         I915_WRITE(GEN6_RPNSWREQ,
6702                    HSW_FREQUENCY(rps->rp1_freq));
6703         I915_WRITE(GEN6_RC_VIDEO_FREQ,
6704                    HSW_FREQUENCY(rps->rp1_freq));
6705         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
6706         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
6707
6708         /* Docs recommend 900MHz, and 300 MHz respectively */
6709         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
6710                    rps->max_freq_softlimit << 24 |
6711                    rps->min_freq_softlimit << 16);
6712
6713         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
6714         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
6715         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
6716         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
6717
6718         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
6719
6720         /* 2: Enable RPS */
6721         I915_WRITE(GEN6_RP_CONTROL,
6722                    GEN6_RP_MEDIA_TURBO |
6723                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
6724                    GEN6_RP_MEDIA_IS_GFX |
6725                    GEN6_RP_ENABLE |
6726                    GEN6_RP_UP_BUSY_AVG |
6727                    GEN6_RP_DOWN_IDLE_AVG);
6728
6729         reset_rps(dev_priv, gen6_set_rps);
6730
6731         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6732 }
6733
6734 static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
6735 {
6736         struct intel_engine_cs *engine;
6737         enum intel_engine_id id;
6738         u32 rc6vids, rc6_mask;
6739         u32 gtfifodbg;
6740         int ret;
6741
6742         I915_WRITE(GEN6_RC_STATE, 0);
6743
6744         /* Clear the DBG now so we don't confuse earlier errors */
6745         gtfifodbg = I915_READ(GTFIFODBG);
6746         if (gtfifodbg) {
6747                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
6748                 I915_WRITE(GTFIFODBG, gtfifodbg);
6749         }
6750
6751         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6752
6753         /* disable the counters and set deterministic thresholds */
6754         I915_WRITE(GEN6_RC_CONTROL, 0);
6755
6756         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
6757         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
6758         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
6759         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
6760         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
6761
6762         for_each_engine(engine, dev_priv, id)
6763                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
6764
6765         I915_WRITE(GEN6_RC_SLEEP, 0);
6766         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
6767         if (IS_IVYBRIDGE(dev_priv))
6768                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
6769         else
6770                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
6771         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
6772         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
6773
6774         /* We don't use those on Haswell */
6775         rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
6776         if (HAS_RC6p(dev_priv))
6777                 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
6778         if (HAS_RC6pp(dev_priv))
6779                 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
6780         I915_WRITE(GEN6_RC_CONTROL,
6781                    rc6_mask |
6782                    GEN6_RC_CTL_EI_MODE(1) |
6783                    GEN6_RC_CTL_HW_ENABLE);
6784
6785         rc6vids = 0;
6786         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
6787         if (IS_GEN6(dev_priv) && ret) {
6788                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
6789         } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
6790                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
6791                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
6792                 rc6vids &= 0xffff00;
6793                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
6794                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
6795                 if (ret)
6796                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
6797         }
6798
6799         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6800 }
6801
6802 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
6803 {
6804         /* Here begins a magic sequence of register writes to enable
6805          * auto-downclocking.
6806          *
6807          * Perhaps there might be some value in exposing these to
6808          * userspace...
6809          */
6810         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
6811
6812         /* Power down if completely idle for over 50ms */
6813         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
6814         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
6815
6816         reset_rps(dev_priv, gen6_set_rps);
6817
6818         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
6819 }
6820
6821 static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
6822 {
6823         struct intel_rps *rps = &dev_priv->gt_pm.rps;
6824         int min_freq = 15;
6825         unsigned int gpu_freq;
6826         unsigned int max_ia_freq, min_ring_freq;
6827         unsigned int max_gpu_freq, min_gpu_freq;
6828         int scaling_factor = 180;
6829         struct cpufreq_policy *policy;
6830
6831         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
6832
6833         policy = cpufreq_cpu_get(0);
6834         if (policy) {
6835                 max_ia_freq = policy->cpuinfo.max_freq;
6836                 cpufreq_cpu_put(policy);
6837         } else {
6838                 /*
6839                  * Default to measured freq if none found, PCU will ensure we
6840                  * don't go over
6841                  */
6842                 max_ia_freq = tsc_khz;
6843         }
6844
6845         /* Convert from kHz to MHz */
6846         max_ia_freq /= 1000;
6847
6848         min_ring_freq = I915_READ(DCLK) & 0xf;
6849         /* convert DDR frequency from units of 266.6MHz to bandwidth */
6850         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
6851
6852         if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
6853                 /* Convert GT frequency to 50 HZ units */
6854                 min_gpu_freq = rps->min_freq / GEN9_FREQ_SCALER;
6855                 max_gpu_freq = rps->max_freq / GEN9_FREQ_SCALER;
6856         } else {
6857                 min_gpu_freq = rps->min_freq;
6858                 max_gpu_freq = rps->max_freq;
6859         }
6860
6861         /*
6862          * For each potential GPU frequency, load a ring frequency we'd like
6863          * to use for memory access.  We do this by specifying the IA frequency
6864          * the PCU should use as a reference to determine the ring frequency.
6865          */
6866         for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
6867                 int diff = max_gpu_freq - gpu_freq;
6868                 unsigned int ia_freq = 0, ring_freq = 0;
6869
6870                 if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
6871                         /*
6872                          * ring_freq = 2 * GT. ring_freq is in 100MHz units
6873                          * No floor required for ring frequency on SKL.
6874                          */
6875                         ring_freq = gpu_freq;
6876                 } else if (INTEL_INFO(dev_priv)->gen >= 8) {
6877                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
6878                         ring_freq = max(min_ring_freq, gpu_freq);
6879                 } else if (IS_HASWELL(dev_priv)) {
6880                         ring_freq = mult_frac(gpu_freq, 5, 4);
6881                         ring_freq = max(min_ring_freq, ring_freq);
6882                         /* leave ia_freq as the default, chosen by cpufreq */
6883                 } else {
6884                         /* On older processors, there is no separate ring
6885                          * clock domain, so in order to boost the bandwidth
6886                          * of the ring, we need to upclock the CPU (ia_freq).
6887                          *
6888                          * For GPU frequencies less than 750MHz,
6889                          * just use the lowest ring freq.
6890                          */
6891                         if (gpu_freq < min_freq)
6892                                 ia_freq = 800;
6893                         else
6894                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
6895                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
6896                 }
6897
6898                 sandybridge_pcode_write(dev_priv,
6899                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
6900                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
6901                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
6902                                         gpu_freq);
6903         }
6904 }
6905
6906 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
6907 {
6908         u32 val, rp0;
6909
6910         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
6911
6912         switch (INTEL_INFO(dev_priv)->sseu.eu_total) {
6913         case 8:
6914                 /* (2 * 4) config */
6915                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
6916                 break;
6917         case 12:
6918                 /* (2 * 6) config */
6919                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
6920                 break;
6921         case 16:
6922                 /* (2 * 8) config */
6923         default:
6924                 /* Setting (2 * 8) Min RP0 for any other combination */
6925                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
6926                 break;
6927         }
6928
6929         rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
6930
6931         return rp0;
6932 }
6933
6934 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
6935 {
6936         u32 val, rpe;
6937
6938         val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
6939         rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
6940
6941         return rpe;
6942 }
6943
6944 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
6945 {
6946         u32 val, rp1;
6947
6948         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
6949         rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
6950
6951         return rp1;
6952 }
6953
6954 static u32 cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
6955 {
6956         u32 val, rpn;
6957
6958         val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
6959         rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
6960                        FB_GFX_FREQ_FUSE_MASK);
6961
6962         return rpn;
6963 }
6964
6965 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
6966 {
6967         u32 val, rp1;
6968
6969         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
6970
6971         rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
6972
6973         return rp1;
6974 }
6975
6976 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
6977 {
6978         u32 val, rp0;
6979
6980         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
6981
6982         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
6983         /* Clamp to max */
6984         rp0 = min_t(u32, rp0, 0xea);
6985
6986         return rp0;
6987 }
6988
6989 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
6990 {
6991         u32 val, rpe;
6992
6993         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
6994         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
6995         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
6996         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
6997
6998         return rpe;
6999 }
7000
7001 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
7002 {
7003         u32 val;
7004
7005         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
7006         /*
7007          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
7008          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
7009          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
7010          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
7011          * to make sure it matches what Punit accepts.
7012          */
7013         return max_t(u32, val, 0xc0);
7014 }
7015
7016 /* Check that the pctx buffer wasn't move under us. */
7017 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
7018 {
7019         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7020
7021         WARN_ON(pctx_addr != dev_priv->dsm.start +
7022                              dev_priv->vlv_pctx->stolen->start);
7023 }
7024
7025
7026 /* Check that the pcbr address is not empty. */
7027 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
7028 {
7029         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
7030
7031         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
7032 }
7033
7034 static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
7035 {
7036         resource_size_t pctx_paddr, paddr;
7037         resource_size_t pctx_size = 32*1024;
7038         u32 pcbr;
7039
7040         pcbr = I915_READ(VLV_PCBR);
7041         if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
7042                 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7043                 paddr = dev_priv->dsm.end + 1 - pctx_size;
7044                 GEM_BUG_ON(paddr > U32_MAX);
7045
7046                 pctx_paddr = (paddr & (~4095));
7047                 I915_WRITE(VLV_PCBR, pctx_paddr);
7048         }
7049
7050         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7051 }
7052
7053 static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
7054 {
7055         struct drm_i915_gem_object *pctx;
7056         resource_size_t pctx_paddr;
7057         resource_size_t pctx_size = 24*1024;
7058         u32 pcbr;
7059
7060         pcbr = I915_READ(VLV_PCBR);
7061         if (pcbr) {
7062                 /* BIOS set it up already, grab the pre-alloc'd space */
7063                 resource_size_t pcbr_offset;
7064
7065                 pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
7066                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
7067                                                                       pcbr_offset,
7068                                                                       I915_GTT_OFFSET_NONE,
7069                                                                       pctx_size);
7070                 goto out;
7071         }
7072
7073         DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
7074
7075         /*
7076          * From the Gunit register HAS:
7077          * The Gfx driver is expected to program this register and ensure
7078          * proper allocation within Gfx stolen memory.  For example, this
7079          * register should be programmed such than the PCBR range does not
7080          * overlap with other ranges, such as the frame buffer, protected
7081          * memory, or any other relevant ranges.
7082          */
7083         pctx = i915_gem_object_create_stolen(dev_priv, pctx_size);
7084         if (!pctx) {
7085                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
7086                 goto out;
7087         }
7088
7089         GEM_BUG_ON(range_overflows_t(u64,
7090                                      dev_priv->dsm.start,
7091                                      pctx->stolen->start,
7092                                      U32_MAX));
7093         pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
7094         I915_WRITE(VLV_PCBR, pctx_paddr);
7095
7096 out:
7097         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
7098         dev_priv->vlv_pctx = pctx;
7099 }
7100
7101 static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
7102 {
7103         if (WARN_ON(!dev_priv->vlv_pctx))
7104                 return;
7105
7106         i915_gem_object_put(dev_priv->vlv_pctx);
7107         dev_priv->vlv_pctx = NULL;
7108 }
7109
7110 static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
7111 {
7112         dev_priv->gt_pm.rps.gpll_ref_freq =
7113                 vlv_get_cck_clock(dev_priv, "GPLL ref",
7114                                   CCK_GPLL_CLOCK_CONTROL,
7115                                   dev_priv->czclk_freq);
7116
7117         DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n",
7118                          dev_priv->gt_pm.rps.gpll_ref_freq);
7119 }
7120
7121 static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
7122 {
7123         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7124         u32 val;
7125
7126         valleyview_setup_pctx(dev_priv);
7127
7128         vlv_init_gpll_ref_freq(dev_priv);
7129
7130         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7131         switch ((val >> 6) & 3) {
7132         case 0:
7133         case 1:
7134                 dev_priv->mem_freq = 800;
7135                 break;
7136         case 2:
7137                 dev_priv->mem_freq = 1066;
7138                 break;
7139         case 3:
7140                 dev_priv->mem_freq = 1333;
7141                 break;
7142         }
7143         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7144
7145         rps->max_freq = valleyview_rps_max_freq(dev_priv);
7146         rps->rp0_freq = rps->max_freq;
7147         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7148                          intel_gpu_freq(dev_priv, rps->max_freq),
7149                          rps->max_freq);
7150
7151         rps->efficient_freq = valleyview_rps_rpe_freq(dev_priv);
7152         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7153                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7154                          rps->efficient_freq);
7155
7156         rps->rp1_freq = valleyview_rps_guar_freq(dev_priv);
7157         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
7158                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7159                          rps->rp1_freq);
7160
7161         rps->min_freq = valleyview_rps_min_freq(dev_priv);
7162         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7163                          intel_gpu_freq(dev_priv, rps->min_freq),
7164                          rps->min_freq);
7165 }
7166
7167 static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
7168 {
7169         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7170         u32 val;
7171
7172         cherryview_setup_pctx(dev_priv);
7173
7174         vlv_init_gpll_ref_freq(dev_priv);
7175
7176         mutex_lock(&dev_priv->sb_lock);
7177         val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
7178         mutex_unlock(&dev_priv->sb_lock);
7179
7180         switch ((val >> 2) & 0x7) {
7181         case 3:
7182                 dev_priv->mem_freq = 2000;
7183                 break;
7184         default:
7185                 dev_priv->mem_freq = 1600;
7186                 break;
7187         }
7188         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
7189
7190         rps->max_freq = cherryview_rps_max_freq(dev_priv);
7191         rps->rp0_freq = rps->max_freq;
7192         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
7193                          intel_gpu_freq(dev_priv, rps->max_freq),
7194                          rps->max_freq);
7195
7196         rps->efficient_freq = cherryview_rps_rpe_freq(dev_priv);
7197         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
7198                          intel_gpu_freq(dev_priv, rps->efficient_freq),
7199                          rps->efficient_freq);
7200
7201         rps->rp1_freq = cherryview_rps_guar_freq(dev_priv);
7202         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
7203                          intel_gpu_freq(dev_priv, rps->rp1_freq),
7204                          rps->rp1_freq);
7205
7206         rps->min_freq = cherryview_rps_min_freq(dev_priv);
7207         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
7208                          intel_gpu_freq(dev_priv, rps->min_freq),
7209                          rps->min_freq);
7210
7211         WARN_ONCE((rps->max_freq | rps->efficient_freq | rps->rp1_freq |
7212                    rps->min_freq) & 1,
7213                   "Odd GPU freq values\n");
7214 }
7215
7216 static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7217 {
7218         valleyview_cleanup_pctx(dev_priv);
7219 }
7220
7221 static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
7222 {
7223         struct intel_engine_cs *engine;
7224         enum intel_engine_id id;
7225         u32 gtfifodbg, rc6_mode, pcbr;
7226
7227         gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
7228                                              GT_FIFO_FREE_ENTRIES_CHV);
7229         if (gtfifodbg) {
7230                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7231                                  gtfifodbg);
7232                 I915_WRITE(GTFIFODBG, gtfifodbg);
7233         }
7234
7235         cherryview_check_pctx(dev_priv);
7236
7237         /* 1a & 1b: Get forcewake during program sequence. Although the driver
7238          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
7239         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7240
7241         /*  Disable RC states. */
7242         I915_WRITE(GEN6_RC_CONTROL, 0);
7243
7244         /* 2a: Program RC6 thresholds.*/
7245         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
7246         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
7247         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
7248
7249         for_each_engine(engine, dev_priv, id)
7250                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7251         I915_WRITE(GEN6_RC_SLEEP, 0);
7252
7253         /* TO threshold set to 500 us ( 0x186 * 1.28 us) */
7254         I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
7255
7256         /* Allows RC6 residency counter to work */
7257         I915_WRITE(VLV_COUNTER_CONTROL,
7258                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7259                                       VLV_MEDIA_RC6_COUNT_EN |
7260                                       VLV_RENDER_RC6_COUNT_EN));
7261
7262         /* For now we assume BIOS is allocating and populating the PCBR  */
7263         pcbr = I915_READ(VLV_PCBR);
7264
7265         /* 3: Enable RC6 */
7266         rc6_mode = 0;
7267         if (pcbr >> VLV_PCBR_ADDR_SHIFT)
7268                 rc6_mode = GEN7_RC_CTL_TO_MODE;
7269         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
7270
7271         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7272 }
7273
7274 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
7275 {
7276         u32 val;
7277
7278         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7279
7280         /* 1: Program defaults and thresholds for RPS*/
7281         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7282         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7283         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7284         I915_WRITE(GEN6_RP_UP_EI, 66000);
7285         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7286
7287         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7288
7289         /* 2: Enable RPS */
7290         I915_WRITE(GEN6_RP_CONTROL,
7291                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7292                    GEN6_RP_MEDIA_IS_GFX |
7293                    GEN6_RP_ENABLE |
7294                    GEN6_RP_UP_BUSY_AVG |
7295                    GEN6_RP_DOWN_IDLE_AVG);
7296
7297         /* Setting Fixed Bias */
7298         val = VLV_OVERRIDE_EN |
7299                   VLV_SOC_TDP_EN |
7300                   CHV_BIAS_CPU_50_SOC_50;
7301         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7302
7303         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7304
7305         /* RPS code assumes GPLL is used */
7306         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7307
7308         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7309         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7310
7311         reset_rps(dev_priv, valleyview_set_rps);
7312
7313         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7314 }
7315
7316 static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
7317 {
7318         struct intel_engine_cs *engine;
7319         enum intel_engine_id id;
7320         u32 gtfifodbg;
7321
7322         valleyview_check_pctx(dev_priv);
7323
7324         gtfifodbg = I915_READ(GTFIFODBG);
7325         if (gtfifodbg) {
7326                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
7327                                  gtfifodbg);
7328                 I915_WRITE(GTFIFODBG, gtfifodbg);
7329         }
7330
7331         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7332
7333         /*  Disable RC states. */
7334         I915_WRITE(GEN6_RC_CONTROL, 0);
7335
7336         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
7337         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
7338         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
7339
7340         for_each_engine(engine, dev_priv, id)
7341                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
7342
7343         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
7344
7345         /* Allows RC6 residency counter to work */
7346         I915_WRITE(VLV_COUNTER_CONTROL,
7347                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
7348                                       VLV_MEDIA_RC0_COUNT_EN |
7349                                       VLV_RENDER_RC0_COUNT_EN |
7350                                       VLV_MEDIA_RC6_COUNT_EN |
7351                                       VLV_RENDER_RC6_COUNT_EN));
7352
7353         I915_WRITE(GEN6_RC_CONTROL,
7354                    GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
7355
7356         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7357 }
7358
7359 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
7360 {
7361         u32 val;
7362
7363         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
7364
7365         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
7366         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
7367         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
7368         I915_WRITE(GEN6_RP_UP_EI, 66000);
7369         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
7370
7371         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
7372
7373         I915_WRITE(GEN6_RP_CONTROL,
7374                    GEN6_RP_MEDIA_TURBO |
7375                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
7376                    GEN6_RP_MEDIA_IS_GFX |
7377                    GEN6_RP_ENABLE |
7378                    GEN6_RP_UP_BUSY_AVG |
7379                    GEN6_RP_DOWN_IDLE_CONT);
7380
7381         /* Setting Fixed Bias */
7382         val = VLV_OVERRIDE_EN |
7383                   VLV_SOC_TDP_EN |
7384                   VLV_BIAS_CPU_125_SOC_875;
7385         vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
7386
7387         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
7388
7389         /* RPS code assumes GPLL is used */
7390         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
7391
7392         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
7393         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
7394
7395         reset_rps(dev_priv, valleyview_set_rps);
7396
7397         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
7398 }
7399
7400 static unsigned long intel_pxfreq(u32 vidfreq)
7401 {
7402         unsigned long freq;
7403         int div = (vidfreq & 0x3f0000) >> 16;
7404         int post = (vidfreq & 0x3000) >> 12;
7405         int pre = (vidfreq & 0x7);
7406
7407         if (!pre)
7408                 return 0;
7409
7410         freq = ((div * 133333) / ((1<<post) * pre));
7411
7412         return freq;
7413 }
7414
7415 static const struct cparams {
7416         u16 i;
7417         u16 t;
7418         u16 m;
7419         u16 c;
7420 } cparams[] = {
7421         { 1, 1333, 301, 28664 },
7422         { 1, 1066, 294, 24460 },
7423         { 1, 800, 294, 25192 },
7424         { 0, 1333, 276, 27605 },
7425         { 0, 1066, 276, 27605 },
7426         { 0, 800, 231, 23784 },
7427 };
7428
7429 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
7430 {
7431         u64 total_count, diff, ret;
7432         u32 count1, count2, count3, m = 0, c = 0;
7433         unsigned long now = jiffies_to_msecs(jiffies), diff1;
7434         int i;
7435
7436         lockdep_assert_held(&mchdev_lock);
7437
7438         diff1 = now - dev_priv->ips.last_time1;
7439
7440         /* Prevent division-by-zero if we are asking too fast.
7441          * Also, we don't get interesting results if we are polling
7442          * faster than once in 10ms, so just return the saved value
7443          * in such cases.
7444          */
7445         if (diff1 <= 10)
7446                 return dev_priv->ips.chipset_power;
7447
7448         count1 = I915_READ(DMIEC);
7449         count2 = I915_READ(DDREC);
7450         count3 = I915_READ(CSIEC);
7451
7452         total_count = count1 + count2 + count3;
7453
7454         /* FIXME: handle per-counter overflow */
7455         if (total_count < dev_priv->ips.last_count1) {
7456                 diff = ~0UL - dev_priv->ips.last_count1;
7457                 diff += total_count;
7458         } else {
7459                 diff = total_count - dev_priv->ips.last_count1;
7460         }
7461
7462         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
7463                 if (cparams[i].i == dev_priv->ips.c_m &&
7464                     cparams[i].t == dev_priv->ips.r_t) {
7465                         m = cparams[i].m;
7466                         c = cparams[i].c;
7467                         break;
7468                 }
7469         }
7470
7471         diff = div_u64(diff, diff1);
7472         ret = ((m * diff) + c);
7473         ret = div_u64(ret, 10);
7474
7475         dev_priv->ips.last_count1 = total_count;
7476         dev_priv->ips.last_time1 = now;
7477
7478         dev_priv->ips.chipset_power = ret;
7479
7480         return ret;
7481 }
7482
7483 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
7484 {
7485         unsigned long val;
7486
7487         if (INTEL_INFO(dev_priv)->gen != 5)
7488                 return 0;
7489
7490         spin_lock_irq(&mchdev_lock);
7491
7492         val = __i915_chipset_val(dev_priv);
7493
7494         spin_unlock_irq(&mchdev_lock);
7495
7496         return val;
7497 }
7498
7499 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
7500 {
7501         unsigned long m, x, b;
7502         u32 tsfs;
7503
7504         tsfs = I915_READ(TSFS);
7505
7506         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
7507         x = I915_READ8(TR1);
7508
7509         b = tsfs & TSFS_INTR_MASK;
7510
7511         return ((m * x) / 127) - b;
7512 }
7513
7514 static int _pxvid_to_vd(u8 pxvid)
7515 {
7516         if (pxvid == 0)
7517                 return 0;
7518
7519         if (pxvid >= 8 && pxvid < 31)
7520                 pxvid = 31;
7521
7522         return (pxvid + 2) * 125;
7523 }
7524
7525 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
7526 {
7527         const int vd = _pxvid_to_vd(pxvid);
7528         const int vm = vd - 1125;
7529
7530         if (INTEL_INFO(dev_priv)->is_mobile)
7531                 return vm > 0 ? vm : 0;
7532
7533         return vd;
7534 }
7535
7536 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
7537 {
7538         u64 now, diff, diffms;
7539         u32 count;
7540
7541         lockdep_assert_held(&mchdev_lock);
7542
7543         now = ktime_get_raw_ns();
7544         diffms = now - dev_priv->ips.last_time2;
7545         do_div(diffms, NSEC_PER_MSEC);
7546
7547         /* Don't divide by 0 */
7548         if (!diffms)
7549                 return;
7550
7551         count = I915_READ(GFXEC);
7552
7553         if (count < dev_priv->ips.last_count2) {
7554                 diff = ~0UL - dev_priv->ips.last_count2;
7555                 diff += count;
7556         } else {
7557                 diff = count - dev_priv->ips.last_count2;
7558         }
7559
7560         dev_priv->ips.last_count2 = count;
7561         dev_priv->ips.last_time2 = now;
7562
7563         /* More magic constants... */
7564         diff = diff * 1181;
7565         diff = div_u64(diff, diffms * 10);
7566         dev_priv->ips.gfx_power = diff;
7567 }
7568
7569 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
7570 {
7571         if (INTEL_INFO(dev_priv)->gen != 5)
7572                 return;
7573
7574         spin_lock_irq(&mchdev_lock);
7575
7576         __i915_update_gfx_val(dev_priv);
7577
7578         spin_unlock_irq(&mchdev_lock);
7579 }
7580
7581 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
7582 {
7583         unsigned long t, corr, state1, corr2, state2;
7584         u32 pxvid, ext_v;
7585
7586         lockdep_assert_held(&mchdev_lock);
7587
7588         pxvid = I915_READ(PXVFREQ(dev_priv->gt_pm.rps.cur_freq));
7589         pxvid = (pxvid >> 24) & 0x7f;
7590         ext_v = pvid_to_extvid(dev_priv, pxvid);
7591
7592         state1 = ext_v;
7593
7594         t = i915_mch_val(dev_priv);
7595
7596         /* Revel in the empirically derived constants */
7597
7598         /* Correction factor in 1/100000 units */
7599         if (t > 80)
7600                 corr = ((t * 2349) + 135940);
7601         else if (t >= 50)
7602                 corr = ((t * 964) + 29317);
7603         else /* < 50 */
7604                 corr = ((t * 301) + 1004);
7605
7606         corr = corr * ((150142 * state1) / 10000 - 78642);
7607         corr /= 100000;
7608         corr2 = (corr * dev_priv->ips.corr);
7609
7610         state2 = (corr2 * state1) / 10000;
7611         state2 /= 100; /* convert to mW */
7612
7613         __i915_update_gfx_val(dev_priv);
7614
7615         return dev_priv->ips.gfx_power + state2;
7616 }
7617
7618 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
7619 {
7620         unsigned long val;
7621
7622         if (INTEL_INFO(dev_priv)->gen != 5)
7623                 return 0;
7624
7625         spin_lock_irq(&mchdev_lock);
7626
7627         val = __i915_gfx_val(dev_priv);
7628
7629         spin_unlock_irq(&mchdev_lock);
7630
7631         return val;
7632 }
7633
7634 /**
7635  * i915_read_mch_val - return value for IPS use
7636  *
7637  * Calculate and return a value for the IPS driver to use when deciding whether
7638  * we have thermal and power headroom to increase CPU or GPU power budget.
7639  */
7640 unsigned long i915_read_mch_val(void)
7641 {
7642         struct drm_i915_private *dev_priv;
7643         unsigned long chipset_val, graphics_val, ret = 0;
7644
7645         spin_lock_irq(&mchdev_lock);
7646         if (!i915_mch_dev)
7647                 goto out_unlock;
7648         dev_priv = i915_mch_dev;
7649
7650         chipset_val = __i915_chipset_val(dev_priv);
7651         graphics_val = __i915_gfx_val(dev_priv);
7652
7653         ret = chipset_val + graphics_val;
7654
7655 out_unlock:
7656         spin_unlock_irq(&mchdev_lock);
7657
7658         return ret;
7659 }
7660 EXPORT_SYMBOL_GPL(i915_read_mch_val);
7661
7662 /**
7663  * i915_gpu_raise - raise GPU frequency limit
7664  *
7665  * Raise the limit; IPS indicates we have thermal headroom.
7666  */
7667 bool i915_gpu_raise(void)
7668 {
7669         struct drm_i915_private *dev_priv;
7670         bool ret = true;
7671
7672         spin_lock_irq(&mchdev_lock);
7673         if (!i915_mch_dev) {
7674                 ret = false;
7675                 goto out_unlock;
7676         }
7677         dev_priv = i915_mch_dev;
7678
7679         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
7680                 dev_priv->ips.max_delay--;
7681
7682 out_unlock:
7683         spin_unlock_irq(&mchdev_lock);
7684
7685         return ret;
7686 }
7687 EXPORT_SYMBOL_GPL(i915_gpu_raise);
7688
7689 /**
7690  * i915_gpu_lower - lower GPU frequency limit
7691  *
7692  * IPS indicates we're close to a thermal limit, so throttle back the GPU
7693  * frequency maximum.
7694  */
7695 bool i915_gpu_lower(void)
7696 {
7697         struct drm_i915_private *dev_priv;
7698         bool ret = true;
7699
7700         spin_lock_irq(&mchdev_lock);
7701         if (!i915_mch_dev) {
7702                 ret = false;
7703                 goto out_unlock;
7704         }
7705         dev_priv = i915_mch_dev;
7706
7707         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
7708                 dev_priv->ips.max_delay++;
7709
7710 out_unlock:
7711         spin_unlock_irq(&mchdev_lock);
7712
7713         return ret;
7714 }
7715 EXPORT_SYMBOL_GPL(i915_gpu_lower);
7716
7717 /**
7718  * i915_gpu_busy - indicate GPU business to IPS
7719  *
7720  * Tell the IPS driver whether or not the GPU is busy.
7721  */
7722 bool i915_gpu_busy(void)
7723 {
7724         bool ret = false;
7725
7726         spin_lock_irq(&mchdev_lock);
7727         if (i915_mch_dev)
7728                 ret = i915_mch_dev->gt.awake;
7729         spin_unlock_irq(&mchdev_lock);
7730
7731         return ret;
7732 }
7733 EXPORT_SYMBOL_GPL(i915_gpu_busy);
7734
7735 /**
7736  * i915_gpu_turbo_disable - disable graphics turbo
7737  *
7738  * Disable graphics turbo by resetting the max frequency and setting the
7739  * current frequency to the default.
7740  */
7741 bool i915_gpu_turbo_disable(void)
7742 {
7743         struct drm_i915_private *dev_priv;
7744         bool ret = true;
7745
7746         spin_lock_irq(&mchdev_lock);
7747         if (!i915_mch_dev) {
7748                 ret = false;
7749                 goto out_unlock;
7750         }
7751         dev_priv = i915_mch_dev;
7752
7753         dev_priv->ips.max_delay = dev_priv->ips.fstart;
7754
7755         if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
7756                 ret = false;
7757
7758 out_unlock:
7759         spin_unlock_irq(&mchdev_lock);
7760
7761         return ret;
7762 }
7763 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
7764
7765 /**
7766  * Tells the intel_ips driver that the i915 driver is now loaded, if
7767  * IPS got loaded first.
7768  *
7769  * This awkward dance is so that neither module has to depend on the
7770  * other in order for IPS to do the appropriate communication of
7771  * GPU turbo limits to i915.
7772  */
7773 static void
7774 ips_ping_for_i915_load(void)
7775 {
7776         void (*link)(void);
7777
7778         link = symbol_get(ips_link_to_i915_driver);
7779         if (link) {
7780                 link();
7781                 symbol_put(ips_link_to_i915_driver);
7782         }
7783 }
7784
7785 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
7786 {
7787         /* We only register the i915 ips part with intel-ips once everything is
7788          * set up, to avoid intel-ips sneaking in and reading bogus values. */
7789         spin_lock_irq(&mchdev_lock);
7790         i915_mch_dev = dev_priv;
7791         spin_unlock_irq(&mchdev_lock);
7792
7793         ips_ping_for_i915_load();
7794 }
7795
7796 void intel_gpu_ips_teardown(void)
7797 {
7798         spin_lock_irq(&mchdev_lock);
7799         i915_mch_dev = NULL;
7800         spin_unlock_irq(&mchdev_lock);
7801 }
7802
7803 static void intel_init_emon(struct drm_i915_private *dev_priv)
7804 {
7805         u32 lcfuse;
7806         u8 pxw[16];
7807         int i;
7808
7809         /* Disable to program */
7810         I915_WRITE(ECR, 0);
7811         POSTING_READ(ECR);
7812
7813         /* Program energy weights for various events */
7814         I915_WRITE(SDEW, 0x15040d00);
7815         I915_WRITE(CSIEW0, 0x007f0000);
7816         I915_WRITE(CSIEW1, 0x1e220004);
7817         I915_WRITE(CSIEW2, 0x04000004);
7818
7819         for (i = 0; i < 5; i++)
7820                 I915_WRITE(PEW(i), 0);
7821         for (i = 0; i < 3; i++)
7822                 I915_WRITE(DEW(i), 0);
7823
7824         /* Program P-state weights to account for frequency power adjustment */
7825         for (i = 0; i < 16; i++) {
7826                 u32 pxvidfreq = I915_READ(PXVFREQ(i));
7827                 unsigned long freq = intel_pxfreq(pxvidfreq);
7828                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
7829                         PXVFREQ_PX_SHIFT;
7830                 unsigned long val;
7831
7832                 val = vid * vid;
7833                 val *= (freq / 1000);
7834                 val *= 255;
7835                 val /= (127*127*900);
7836                 if (val > 0xff)
7837                         DRM_ERROR("bad pxval: %ld\n", val);
7838                 pxw[i] = val;
7839         }
7840         /* Render standby states get 0 weight */
7841         pxw[14] = 0;
7842         pxw[15] = 0;
7843
7844         for (i = 0; i < 4; i++) {
7845                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
7846                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
7847                 I915_WRITE(PXW(i), val);
7848         }
7849
7850         /* Adjust magic regs to magic values (more experimental results) */
7851         I915_WRITE(OGW0, 0);
7852         I915_WRITE(OGW1, 0);
7853         I915_WRITE(EG0, 0x00007f00);
7854         I915_WRITE(EG1, 0x0000000e);
7855         I915_WRITE(EG2, 0x000e0000);
7856         I915_WRITE(EG3, 0x68000300);
7857         I915_WRITE(EG4, 0x42000000);
7858         I915_WRITE(EG5, 0x00140031);
7859         I915_WRITE(EG6, 0);
7860         I915_WRITE(EG7, 0);
7861
7862         for (i = 0; i < 8; i++)
7863                 I915_WRITE(PXWL(i), 0);
7864
7865         /* Enable PMON + select events */
7866         I915_WRITE(ECR, 0x80000019);
7867
7868         lcfuse = I915_READ(LCFUSE02);
7869
7870         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
7871 }
7872
7873 void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
7874 {
7875         struct intel_rps *rps = &dev_priv->gt_pm.rps;
7876
7877         /*
7878          * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
7879          * requirement.
7880          */
7881         if (!sanitize_rc6(dev_priv)) {
7882                 DRM_INFO("RC6 disabled, disabling runtime PM support\n");
7883                 intel_runtime_pm_get(dev_priv);
7884         }
7885
7886         mutex_lock(&dev_priv->pcu_lock);
7887
7888         /* Initialize RPS limits (for userspace) */
7889         if (IS_CHERRYVIEW(dev_priv))
7890                 cherryview_init_gt_powersave(dev_priv);
7891         else if (IS_VALLEYVIEW(dev_priv))
7892                 valleyview_init_gt_powersave(dev_priv);
7893         else if (INTEL_GEN(dev_priv) >= 6)
7894                 gen6_init_rps_frequencies(dev_priv);
7895
7896         /* Derive initial user preferences/limits from the hardware limits */
7897         rps->idle_freq = rps->min_freq;
7898         rps->cur_freq = rps->idle_freq;
7899
7900         rps->max_freq_softlimit = rps->max_freq;
7901         rps->min_freq_softlimit = rps->min_freq;
7902
7903         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7904                 rps->min_freq_softlimit =
7905                         max_t(int,
7906                               rps->efficient_freq,
7907                               intel_freq_opcode(dev_priv, 450));
7908
7909         /* After setting max-softlimit, find the overclock max freq */
7910         if (IS_GEN6(dev_priv) ||
7911             IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
7912                 u32 params = 0;
7913
7914                 sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &params);
7915                 if (params & BIT(31)) { /* OC supported */
7916                         DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n",
7917                                          (rps->max_freq & 0xff) * 50,
7918                                          (params & 0xff) * 50);
7919                         rps->max_freq = params & 0xff;
7920                 }
7921         }
7922
7923         /* Finally allow us to boost to max by default */
7924         rps->boost_freq = rps->max_freq;
7925
7926         mutex_unlock(&dev_priv->pcu_lock);
7927 }
7928
7929 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
7930 {
7931         if (IS_VALLEYVIEW(dev_priv))
7932                 valleyview_cleanup_gt_powersave(dev_priv);
7933
7934         if (!HAS_RC6(dev_priv))
7935                 intel_runtime_pm_put(dev_priv);
7936 }
7937
7938 /**
7939  * intel_suspend_gt_powersave - suspend PM work and helper threads
7940  * @dev_priv: i915 device
7941  *
7942  * We don't want to disable RC6 or other features here, we just want
7943  * to make sure any work we've queued has finished and won't bother
7944  * us while we're suspended.
7945  */
7946 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
7947 {
7948         if (INTEL_GEN(dev_priv) < 6)
7949                 return;
7950
7951         /* gen6_rps_idle() will be called later to disable interrupts */
7952 }
7953
7954 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
7955 {
7956         dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
7957         dev_priv->gt_pm.rc6.enabled = true; /* force RC6 disabling */
7958         intel_disable_gt_powersave(dev_priv);
7959
7960         gen6_reset_rps_interrupts(dev_priv);
7961 }
7962
7963 static inline void intel_disable_llc_pstate(struct drm_i915_private *i915)
7964 {
7965         lockdep_assert_held(&i915->pcu_lock);
7966
7967         if (!i915->gt_pm.llc_pstate.enabled)
7968                 return;
7969
7970         /* Currently there is no HW configuration to be done to disable. */
7971
7972         i915->gt_pm.llc_pstate.enabled = false;
7973 }
7974
7975 static void intel_disable_rc6(struct drm_i915_private *dev_priv)
7976 {
7977         lockdep_assert_held(&dev_priv->pcu_lock);
7978
7979         if (!dev_priv->gt_pm.rc6.enabled)
7980                 return;
7981
7982         if (INTEL_GEN(dev_priv) >= 9)
7983                 gen9_disable_rc6(dev_priv);
7984         else if (IS_CHERRYVIEW(dev_priv))
7985                 cherryview_disable_rc6(dev_priv);
7986         else if (IS_VALLEYVIEW(dev_priv))
7987                 valleyview_disable_rc6(dev_priv);
7988         else if (INTEL_GEN(dev_priv) >= 6)
7989                 gen6_disable_rc6(dev_priv);
7990
7991         dev_priv->gt_pm.rc6.enabled = false;
7992 }
7993
7994 static void intel_disable_rps(struct drm_i915_private *dev_priv)
7995 {
7996         lockdep_assert_held(&dev_priv->pcu_lock);
7997
7998         if (!dev_priv->gt_pm.rps.enabled)
7999                 return;
8000
8001         if (INTEL_GEN(dev_priv) >= 9)
8002                 gen9_disable_rps(dev_priv);
8003         else if (IS_CHERRYVIEW(dev_priv))
8004                 cherryview_disable_rps(dev_priv);
8005         else if (IS_VALLEYVIEW(dev_priv))
8006                 valleyview_disable_rps(dev_priv);
8007         else if (INTEL_GEN(dev_priv) >= 6)
8008                 gen6_disable_rps(dev_priv);
8009         else if (IS_IRONLAKE_M(dev_priv))
8010                 ironlake_disable_drps(dev_priv);
8011
8012         dev_priv->gt_pm.rps.enabled = false;
8013 }
8014
8015 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
8016 {
8017         mutex_lock(&dev_priv->pcu_lock);
8018
8019         intel_disable_rc6(dev_priv);
8020         intel_disable_rps(dev_priv);
8021         if (HAS_LLC(dev_priv))
8022                 intel_disable_llc_pstate(dev_priv);
8023
8024         mutex_unlock(&dev_priv->pcu_lock);
8025 }
8026
8027 static inline void intel_enable_llc_pstate(struct drm_i915_private *i915)
8028 {
8029         lockdep_assert_held(&i915->pcu_lock);
8030
8031         if (i915->gt_pm.llc_pstate.enabled)
8032                 return;
8033
8034         gen6_update_ring_freq(i915);
8035
8036         i915->gt_pm.llc_pstate.enabled = true;
8037 }
8038
8039 static void intel_enable_rc6(struct drm_i915_private *dev_priv)
8040 {
8041         lockdep_assert_held(&dev_priv->pcu_lock);
8042
8043         if (dev_priv->gt_pm.rc6.enabled)
8044                 return;
8045
8046         if (IS_CHERRYVIEW(dev_priv))
8047                 cherryview_enable_rc6(dev_priv);
8048         else if (IS_VALLEYVIEW(dev_priv))
8049                 valleyview_enable_rc6(dev_priv);
8050         else if (INTEL_GEN(dev_priv) >= 9)
8051                 gen9_enable_rc6(dev_priv);
8052         else if (IS_BROADWELL(dev_priv))
8053                 gen8_enable_rc6(dev_priv);
8054         else if (INTEL_GEN(dev_priv) >= 6)
8055                 gen6_enable_rc6(dev_priv);
8056
8057         dev_priv->gt_pm.rc6.enabled = true;
8058 }
8059
8060 static void intel_enable_rps(struct drm_i915_private *dev_priv)
8061 {
8062         struct intel_rps *rps = &dev_priv->gt_pm.rps;
8063
8064         lockdep_assert_held(&dev_priv->pcu_lock);
8065
8066         if (rps->enabled)
8067                 return;
8068
8069         if (IS_CHERRYVIEW(dev_priv)) {
8070                 cherryview_enable_rps(dev_priv);
8071         } else if (IS_VALLEYVIEW(dev_priv)) {
8072                 valleyview_enable_rps(dev_priv);
8073         } else if (INTEL_GEN(dev_priv) >= 9) {
8074                 gen9_enable_rps(dev_priv);
8075         } else if (IS_BROADWELL(dev_priv)) {
8076                 gen8_enable_rps(dev_priv);
8077         } else if (INTEL_GEN(dev_priv) >= 6) {
8078                 gen6_enable_rps(dev_priv);
8079         } else if (IS_IRONLAKE_M(dev_priv)) {
8080                 ironlake_enable_drps(dev_priv);
8081                 intel_init_emon(dev_priv);
8082         }
8083
8084         WARN_ON(rps->max_freq < rps->min_freq);
8085         WARN_ON(rps->idle_freq > rps->max_freq);
8086
8087         WARN_ON(rps->efficient_freq < rps->min_freq);
8088         WARN_ON(rps->efficient_freq > rps->max_freq);
8089
8090         rps->enabled = true;
8091 }
8092
8093 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
8094 {
8095         /* Powersaving is controlled by the host when inside a VM */
8096         if (intel_vgpu_active(dev_priv))
8097                 return;
8098
8099         mutex_lock(&dev_priv->pcu_lock);
8100
8101         if (HAS_RC6(dev_priv))
8102                 intel_enable_rc6(dev_priv);
8103         intel_enable_rps(dev_priv);
8104         if (HAS_LLC(dev_priv))
8105                 intel_enable_llc_pstate(dev_priv);
8106
8107         mutex_unlock(&dev_priv->pcu_lock);
8108 }
8109
8110 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
8111 {
8112         /*
8113          * On Ibex Peak and Cougar Point, we need to disable clock
8114          * gating for the panel power sequencer or it will fail to
8115          * start up when no ports are active.
8116          */
8117         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
8118 }
8119
8120 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
8121 {
8122         enum pipe pipe;
8123
8124         for_each_pipe(dev_priv, pipe) {
8125                 I915_WRITE(DSPCNTR(pipe),
8126                            I915_READ(DSPCNTR(pipe)) |
8127                            DISPPLANE_TRICKLE_FEED_DISABLE);
8128
8129                 I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
8130                 POSTING_READ(DSPSURF(pipe));
8131         }
8132 }
8133
8134 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
8135 {
8136         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8137
8138         /*
8139          * Required for FBC
8140          * WaFbcDisableDpfcClockGating:ilk
8141          */
8142         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
8143                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
8144                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
8145
8146         I915_WRITE(PCH_3DCGDIS0,
8147                    MARIUNIT_CLOCK_GATE_DISABLE |
8148                    SVSMUNIT_CLOCK_GATE_DISABLE);
8149         I915_WRITE(PCH_3DCGDIS1,
8150                    VFMUNIT_CLOCK_GATE_DISABLE);
8151
8152         /*
8153          * According to the spec the following bits should be set in
8154          * order to enable memory self-refresh
8155          * The bit 22/21 of 0x42004
8156          * The bit 5 of 0x42020
8157          * The bit 15 of 0x45000
8158          */
8159         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8160                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
8161                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
8162         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
8163         I915_WRITE(DISP_ARB_CTL,
8164                    (I915_READ(DISP_ARB_CTL) |
8165                     DISP_FBC_WM_DIS));
8166
8167         /*
8168          * Based on the document from hardware guys the following bits
8169          * should be set unconditionally in order to enable FBC.
8170          * The bit 22 of 0x42000
8171          * The bit 22 of 0x42004
8172          * The bit 7,8,9 of 0x42020.
8173          */
8174         if (IS_IRONLAKE_M(dev_priv)) {
8175                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
8176                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
8177                            I915_READ(ILK_DISPLAY_CHICKEN1) |
8178                            ILK_FBCQ_DIS);
8179                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
8180                            I915_READ(ILK_DISPLAY_CHICKEN2) |
8181                            ILK_DPARB_GATE);
8182         }
8183
8184         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8185
8186         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8187                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8188                    ILK_ELPIN_409_SELECT);
8189         I915_WRITE(_3D_CHICKEN2,
8190                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
8191                    _3D_CHICKEN2_WM_READ_PIPELINED);
8192
8193         /* WaDisableRenderCachePipelinedFlush:ilk */
8194         I915_WRITE(CACHE_MODE_0,
8195                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8196
8197         /* WaDisable_RenderCache_OperationalFlush:ilk */
8198         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8199
8200         g4x_disable_trickle_feed(dev_priv);
8201
8202         ibx_init_clock_gating(dev_priv);
8203 }
8204
8205 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
8206 {
8207         int pipe;
8208         uint32_t val;
8209
8210         /*
8211          * On Ibex Peak and Cougar Point, we need to disable clock
8212          * gating for the panel power sequencer or it will fail to
8213          * start up when no ports are active.
8214          */
8215         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
8216                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
8217                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
8218         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
8219                    DPLS_EDP_PPS_FIX_DIS);
8220         /* The below fixes the weird display corruption, a few pixels shifted
8221          * downward, on (only) LVDS of some HP laptops with IVY.
8222          */
8223         for_each_pipe(dev_priv, pipe) {
8224                 val = I915_READ(TRANS_CHICKEN2(pipe));
8225                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
8226                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8227                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
8228                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
8229                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
8230                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
8231                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
8232                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
8233         }
8234         /* WADP0ClockGatingDisable */
8235         for_each_pipe(dev_priv, pipe) {
8236                 I915_WRITE(TRANS_CHICKEN1(pipe),
8237                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8238         }
8239 }
8240
8241 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
8242 {
8243         uint32_t tmp;
8244
8245         tmp = I915_READ(MCH_SSKPD);
8246         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
8247                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
8248                               tmp);
8249 }
8250
8251 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
8252 {
8253         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
8254
8255         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
8256
8257         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8258                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8259                    ILK_ELPIN_409_SELECT);
8260
8261         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
8262         I915_WRITE(_3D_CHICKEN,
8263                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
8264
8265         /* WaDisable_RenderCache_OperationalFlush:snb */
8266         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8267
8268         /*
8269          * BSpec recoomends 8x4 when MSAA is used,
8270          * however in practice 16x4 seems fastest.
8271          *
8272          * Note that PS/WM thread counts depend on the WIZ hashing
8273          * disable bit, which we don't touch here, but it's good
8274          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8275          */
8276         I915_WRITE(GEN6_GT_MODE,
8277                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8278
8279         I915_WRITE(CACHE_MODE_0,
8280                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
8281
8282         I915_WRITE(GEN6_UCGCTL1,
8283                    I915_READ(GEN6_UCGCTL1) |
8284                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
8285                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
8286
8287         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
8288          * gating disable must be set.  Failure to set it results in
8289          * flickering pixels due to Z write ordering failures after
8290          * some amount of runtime in the Mesa "fire" demo, and Unigine
8291          * Sanctuary and Tropics, and apparently anything else with
8292          * alpha test or pixel discard.
8293          *
8294          * According to the spec, bit 11 (RCCUNIT) must also be set,
8295          * but we didn't debug actual testcases to find it out.
8296          *
8297          * WaDisableRCCUnitClockGating:snb
8298          * WaDisableRCPBUnitClockGating:snb
8299          */
8300         I915_WRITE(GEN6_UCGCTL2,
8301                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
8302                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
8303
8304         /* WaStripsFansDisableFastClipPerformanceFix:snb */
8305         I915_WRITE(_3D_CHICKEN3,
8306                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
8307
8308         /*
8309          * Bspec says:
8310          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
8311          * 3DSTATE_SF number of SF output attributes is more than 16."
8312          */
8313         I915_WRITE(_3D_CHICKEN3,
8314                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
8315
8316         /*
8317          * According to the spec the following bits should be
8318          * set in order to enable memory self-refresh and fbc:
8319          * The bit21 and bit22 of 0x42000
8320          * The bit21 and bit22 of 0x42004
8321          * The bit5 and bit7 of 0x42020
8322          * The bit14 of 0x70180
8323          * The bit14 of 0x71180
8324          *
8325          * WaFbcAsynchFlipDisableFbcQueue:snb
8326          */
8327         I915_WRITE(ILK_DISPLAY_CHICKEN1,
8328                    I915_READ(ILK_DISPLAY_CHICKEN1) |
8329                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
8330         I915_WRITE(ILK_DISPLAY_CHICKEN2,
8331                    I915_READ(ILK_DISPLAY_CHICKEN2) |
8332                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
8333         I915_WRITE(ILK_DSPCLK_GATE_D,
8334                    I915_READ(ILK_DSPCLK_GATE_D) |
8335                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
8336                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
8337
8338         g4x_disable_trickle_feed(dev_priv);
8339
8340         cpt_init_clock_gating(dev_priv);
8341
8342         gen6_check_mch_setup(dev_priv);
8343 }
8344
8345 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
8346 {
8347         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
8348
8349         /*
8350          * WaVSThreadDispatchOverride:ivb,vlv
8351          *
8352          * This actually overrides the dispatch
8353          * mode for all thread types.
8354          */
8355         reg &= ~GEN7_FF_SCHED_MASK;
8356         reg |= GEN7_FF_TS_SCHED_HW;
8357         reg |= GEN7_FF_VS_SCHED_HW;
8358         reg |= GEN7_FF_DS_SCHED_HW;
8359
8360         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
8361 }
8362
8363 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
8364 {
8365         /*
8366          * TODO: this bit should only be enabled when really needed, then
8367          * disabled when not needed anymore in order to save power.
8368          */
8369         if (HAS_PCH_LPT_LP(dev_priv))
8370                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
8371                            I915_READ(SOUTH_DSPCLK_GATE_D) |
8372                            PCH_LP_PARTITION_LEVEL_DISABLE);
8373
8374         /* WADPOClockGatingDisable:hsw */
8375         I915_WRITE(TRANS_CHICKEN1(PIPE_A),
8376                    I915_READ(TRANS_CHICKEN1(PIPE_A)) |
8377                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
8378 }
8379
8380 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
8381 {
8382         if (HAS_PCH_LPT_LP(dev_priv)) {
8383                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
8384
8385                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
8386                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
8387         }
8388 }
8389
8390 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
8391                                    int general_prio_credits,
8392                                    int high_prio_credits)
8393 {
8394         u32 misccpctl;
8395         u32 val;
8396
8397         /* WaTempDisableDOPClkGating:bdw */
8398         misccpctl = I915_READ(GEN7_MISCCPCTL);
8399         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
8400
8401         val = I915_READ(GEN8_L3SQCREG1);
8402         val &= ~L3_PRIO_CREDITS_MASK;
8403         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
8404         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
8405         I915_WRITE(GEN8_L3SQCREG1, val);
8406
8407         /*
8408          * Wait at least 100 clocks before re-enabling clock gating.
8409          * See the definition of L3SQCREG1 in BSpec.
8410          */
8411         POSTING_READ(GEN8_L3SQCREG1);
8412         udelay(1);
8413         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
8414 }
8415
8416 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
8417 {
8418         if (!HAS_PCH_CNP(dev_priv))
8419                 return;
8420
8421         /* Display WA #1181: cnp */
8422         I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
8423                    CNP_PWM_CGE_GATING_DISABLE);
8424 }
8425
8426 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
8427 {
8428         u32 val;
8429         cnp_init_clock_gating(dev_priv);
8430
8431         /* This is not an Wa. Enable for better image quality */
8432         I915_WRITE(_3D_CHICKEN3,
8433                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
8434
8435         /* WaEnableChickenDCPR:cnl */
8436         I915_WRITE(GEN8_CHICKEN_DCPR_1,
8437                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
8438
8439         /* WaFbcWakeMemOn:cnl */
8440         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
8441                    DISP_FBC_MEMORY_WAKE);
8442
8443         val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
8444         /* ReadHitWriteOnlyDisable:cnl */
8445         val |= RCCUNIT_CLKGATE_DIS;
8446         /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
8447         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
8448                 val |= SARBUNIT_CLKGATE_DIS;
8449         I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
8450
8451         /* WaDisableVFclkgate:cnl */
8452         val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
8453         val |= VFUNIT_CLKGATE_DIS;
8454         I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
8455 }
8456
8457 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
8458 {
8459         cnp_init_clock_gating(dev_priv);
8460         gen9_init_clock_gating(dev_priv);
8461
8462         /* WaFbcNukeOnHostModify:cfl */
8463         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8464                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8465 }
8466
8467 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
8468 {
8469         gen9_init_clock_gating(dev_priv);
8470
8471         /* WaDisableSDEUnitClockGating:kbl */
8472         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8473                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8474                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8475
8476         /* WaDisableGamClockGating:kbl */
8477         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
8478                 I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
8479                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
8480
8481         /* WaFbcNukeOnHostModify:kbl */
8482         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8483                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8484 }
8485
8486 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
8487 {
8488         gen9_init_clock_gating(dev_priv);
8489
8490         /* WAC6entrylatency:skl */
8491         I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
8492                    FBC_LLC_FULLY_OPEN);
8493
8494         /* WaFbcNukeOnHostModify:skl */
8495         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
8496                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
8497 }
8498
8499 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
8500 {
8501         /* The GTT cache must be disabled if the system is using 2M pages. */
8502         bool can_use_gtt_cache = !HAS_PAGE_SIZES(dev_priv,
8503                                                  I915_GTT_PAGE_SIZE_2M);
8504         enum pipe pipe;
8505
8506         /* WaSwitchSolVfFArbitrationPriority:bdw */
8507         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8508
8509         /* WaPsrDPAMaskVBlankInSRD:bdw */
8510         I915_WRITE(CHICKEN_PAR1_1,
8511                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
8512
8513         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
8514         for_each_pipe(dev_priv, pipe) {
8515                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
8516                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
8517                            BDW_DPRS_MASK_VBLANK_SRD);
8518         }
8519
8520         /* WaVSRefCountFullforceMissDisable:bdw */
8521         /* WaDSRefCountFullforceMissDisable:bdw */
8522         I915_WRITE(GEN7_FF_THREAD_MODE,
8523                    I915_READ(GEN7_FF_THREAD_MODE) &
8524                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
8525
8526         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
8527                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
8528
8529         /* WaDisableSDEUnitClockGating:bdw */
8530         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8531                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8532
8533         /* WaProgramL3SqcReg1Default:bdw */
8534         gen8_set_l3sqc_credits(dev_priv, 30, 2);
8535
8536         /* WaGttCachingOffByDefault:bdw */
8537         I915_WRITE(HSW_GTT_CACHE_EN, can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
8538
8539         /* WaKVMNotificationOnConfigChange:bdw */
8540         I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
8541                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
8542
8543         lpt_init_clock_gating(dev_priv);
8544
8545         /* WaDisableDopClockGating:bdw
8546          *
8547          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
8548          * clock gating.
8549          */
8550         I915_WRITE(GEN6_UCGCTL1,
8551                    I915_READ(GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
8552 }
8553
8554 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
8555 {
8556         /* L3 caching of data atomics doesn't work -- disable it. */
8557         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
8558         I915_WRITE(HSW_ROW_CHICKEN3,
8559                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
8560
8561         /* This is required by WaCatErrorRejectionIssue:hsw */
8562         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8563                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8564                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8565
8566         /* WaVSRefCountFullforceMissDisable:hsw */
8567         I915_WRITE(GEN7_FF_THREAD_MODE,
8568                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
8569
8570         /* WaDisable_RenderCache_OperationalFlush:hsw */
8571         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8572
8573         /* enable HiZ Raw Stall Optimization */
8574         I915_WRITE(CACHE_MODE_0_GEN7,
8575                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
8576
8577         /* WaDisable4x2SubspanOptimization:hsw */
8578         I915_WRITE(CACHE_MODE_1,
8579                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
8580
8581         /*
8582          * BSpec recommends 8x4 when MSAA is used,
8583          * however in practice 16x4 seems fastest.
8584          *
8585          * Note that PS/WM thread counts depend on the WIZ hashing
8586          * disable bit, which we don't touch here, but it's good
8587          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8588          */
8589         I915_WRITE(GEN7_GT_MODE,
8590                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8591
8592         /* WaSampleCChickenBitEnable:hsw */
8593         I915_WRITE(HALF_SLICE_CHICKEN3,
8594                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
8595
8596         /* WaSwitchSolVfFArbitrationPriority:hsw */
8597         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
8598
8599         lpt_init_clock_gating(dev_priv);
8600 }
8601
8602 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
8603 {
8604         uint32_t snpcr;
8605
8606         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
8607
8608         /* WaDisableEarlyCull:ivb */
8609         I915_WRITE(_3D_CHICKEN3,
8610                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
8611
8612         /* WaDisableBackToBackFlipFix:ivb */
8613         I915_WRITE(IVB_CHICKEN3,
8614                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
8615                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
8616
8617         /* WaDisablePSDDualDispatchEnable:ivb */
8618         if (IS_IVB_GT1(dev_priv))
8619                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
8620                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
8621
8622         /* WaDisable_RenderCache_OperationalFlush:ivb */
8623         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8624
8625         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
8626         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
8627                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
8628
8629         /* WaApplyL3ControlAndL3ChickenMode:ivb */
8630         I915_WRITE(GEN7_L3CNTLREG1,
8631                         GEN7_WA_FOR_GEN7_L3_CONTROL);
8632         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
8633                    GEN7_WA_L3_CHICKEN_MODE);
8634         if (IS_IVB_GT1(dev_priv))
8635                 I915_WRITE(GEN7_ROW_CHICKEN2,
8636                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8637         else {
8638                 /* must write both registers */
8639                 I915_WRITE(GEN7_ROW_CHICKEN2,
8640                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8641                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
8642                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8643         }
8644
8645         /* WaForceL3Serialization:ivb */
8646         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
8647                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
8648
8649         /*
8650          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
8651          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
8652          */
8653         I915_WRITE(GEN6_UCGCTL2,
8654                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
8655
8656         /* This is required by WaCatErrorRejectionIssue:ivb */
8657         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8658                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8659                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8660
8661         g4x_disable_trickle_feed(dev_priv);
8662
8663         gen7_setup_fixed_func_scheduler(dev_priv);
8664
8665         if (0) { /* causes HiZ corruption on ivb:gt1 */
8666                 /* enable HiZ Raw Stall Optimization */
8667                 I915_WRITE(CACHE_MODE_0_GEN7,
8668                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
8669         }
8670
8671         /* WaDisable4x2SubspanOptimization:ivb */
8672         I915_WRITE(CACHE_MODE_1,
8673                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
8674
8675         /*
8676          * BSpec recommends 8x4 when MSAA is used,
8677          * however in practice 16x4 seems fastest.
8678          *
8679          * Note that PS/WM thread counts depend on the WIZ hashing
8680          * disable bit, which we don't touch here, but it's good
8681          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8682          */
8683         I915_WRITE(GEN7_GT_MODE,
8684                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8685
8686         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
8687         snpcr &= ~GEN6_MBC_SNPCR_MASK;
8688         snpcr |= GEN6_MBC_SNPCR_MED;
8689         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
8690
8691         if (!HAS_PCH_NOP(dev_priv))
8692                 cpt_init_clock_gating(dev_priv);
8693
8694         gen6_check_mch_setup(dev_priv);
8695 }
8696
8697 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
8698 {
8699         /* WaDisableEarlyCull:vlv */
8700         I915_WRITE(_3D_CHICKEN3,
8701                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
8702
8703         /* WaDisableBackToBackFlipFix:vlv */
8704         I915_WRITE(IVB_CHICKEN3,
8705                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
8706                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
8707
8708         /* WaPsdDispatchEnable:vlv */
8709         /* WaDisablePSDDualDispatchEnable:vlv */
8710         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
8711                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
8712                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
8713
8714         /* WaDisable_RenderCache_OperationalFlush:vlv */
8715         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8716
8717         /* WaForceL3Serialization:vlv */
8718         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
8719                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
8720
8721         /* WaDisableDopClockGating:vlv */
8722         I915_WRITE(GEN7_ROW_CHICKEN2,
8723                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
8724
8725         /* This is required by WaCatErrorRejectionIssue:vlv */
8726         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
8727                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
8728                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
8729
8730         gen7_setup_fixed_func_scheduler(dev_priv);
8731
8732         /*
8733          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
8734          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
8735          */
8736         I915_WRITE(GEN6_UCGCTL2,
8737                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
8738
8739         /* WaDisableL3Bank2xClockGate:vlv
8740          * Disabling L3 clock gating- MMIO 940c[25] = 1
8741          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
8742         I915_WRITE(GEN7_UCGCTL4,
8743                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
8744
8745         /*
8746          * BSpec says this must be set, even though
8747          * WaDisable4x2SubspanOptimization isn't listed for VLV.
8748          */
8749         I915_WRITE(CACHE_MODE_1,
8750                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
8751
8752         /*
8753          * BSpec recommends 8x4 when MSAA is used,
8754          * however in practice 16x4 seems fastest.
8755          *
8756          * Note that PS/WM thread counts depend on the WIZ hashing
8757          * disable bit, which we don't touch here, but it's good
8758          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
8759          */
8760         I915_WRITE(GEN7_GT_MODE,
8761                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
8762
8763         /*
8764          * WaIncreaseL3CreditsForVLVB0:vlv
8765          * This is the hardware default actually.
8766          */
8767         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
8768
8769         /*
8770          * WaDisableVLVClockGating_VBIIssue:vlv
8771          * Disable clock gating on th GCFG unit to prevent a delay
8772          * in the reporting of vblank events.
8773          */
8774         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
8775 }
8776
8777 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
8778 {
8779         /* WaVSRefCountFullforceMissDisable:chv */
8780         /* WaDSRefCountFullforceMissDisable:chv */
8781         I915_WRITE(GEN7_FF_THREAD_MODE,
8782                    I915_READ(GEN7_FF_THREAD_MODE) &
8783                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
8784
8785         /* WaDisableSemaphoreAndSyncFlipWait:chv */
8786         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
8787                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
8788
8789         /* WaDisableCSUnitClockGating:chv */
8790         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
8791                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
8792
8793         /* WaDisableSDEUnitClockGating:chv */
8794         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
8795                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
8796
8797         /*
8798          * WaProgramL3SqcReg1Default:chv
8799          * See gfxspecs/Related Documents/Performance Guide/
8800          * LSQC Setting Recommendations.
8801          */
8802         gen8_set_l3sqc_credits(dev_priv, 38, 2);
8803
8804         /*
8805          * GTT cache may not work with big pages, so if those
8806          * are ever enabled GTT cache may need to be disabled.
8807          */
8808         I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
8809 }
8810
8811 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
8812 {
8813         uint32_t dspclk_gate;
8814
8815         I915_WRITE(RENCLK_GATE_D1, 0);
8816         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
8817                    GS_UNIT_CLOCK_GATE_DISABLE |
8818                    CL_UNIT_CLOCK_GATE_DISABLE);
8819         I915_WRITE(RAMCLK_GATE_D, 0);
8820         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
8821                 OVRUNIT_CLOCK_GATE_DISABLE |
8822                 OVCUNIT_CLOCK_GATE_DISABLE;
8823         if (IS_GM45(dev_priv))
8824                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
8825         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
8826
8827         /* WaDisableRenderCachePipelinedFlush */
8828         I915_WRITE(CACHE_MODE_0,
8829                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
8830
8831         /* WaDisable_RenderCache_OperationalFlush:g4x */
8832         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8833
8834         g4x_disable_trickle_feed(dev_priv);
8835 }
8836
8837 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
8838 {
8839         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
8840         I915_WRITE(RENCLK_GATE_D2, 0);
8841         I915_WRITE(DSPCLK_GATE_D, 0);
8842         I915_WRITE(RAMCLK_GATE_D, 0);
8843         I915_WRITE16(DEUC, 0);
8844         I915_WRITE(MI_ARB_STATE,
8845                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
8846
8847         /* WaDisable_RenderCache_OperationalFlush:gen4 */
8848         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8849 }
8850
8851 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
8852 {
8853         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
8854                    I965_RCC_CLOCK_GATE_DISABLE |
8855                    I965_RCPB_CLOCK_GATE_DISABLE |
8856                    I965_ISC_CLOCK_GATE_DISABLE |
8857                    I965_FBC_CLOCK_GATE_DISABLE);
8858         I915_WRITE(RENCLK_GATE_D2, 0);
8859         I915_WRITE(MI_ARB_STATE,
8860                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
8861
8862         /* WaDisable_RenderCache_OperationalFlush:gen4 */
8863         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
8864 }
8865
8866 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
8867 {
8868         u32 dstate = I915_READ(D_STATE);
8869
8870         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
8871                 DSTATE_DOT_CLOCK_GATING;
8872         I915_WRITE(D_STATE, dstate);
8873
8874         if (IS_PINEVIEW(dev_priv))
8875                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
8876
8877         /* IIR "flip pending" means done if this bit is set */
8878         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
8879
8880         /* interrupts should cause a wake up from C3 */
8881         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
8882
8883         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
8884         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
8885
8886         I915_WRITE(MI_ARB_STATE,
8887                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
8888 }
8889
8890 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
8891 {
8892         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
8893
8894         /* interrupts should cause a wake up from C3 */
8895         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
8896                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
8897
8898         I915_WRITE(MEM_MODE,
8899                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
8900 }
8901
8902 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
8903 {
8904         I915_WRITE(MEM_MODE,
8905                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
8906                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
8907 }
8908
8909 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
8910 {
8911         dev_priv->display.init_clock_gating(dev_priv);
8912 }
8913
8914 void intel_suspend_hw(struct drm_i915_private *dev_priv)
8915 {
8916         if (HAS_PCH_LPT(dev_priv))
8917                 lpt_suspend_hw(dev_priv);
8918 }
8919
8920 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
8921 {
8922         DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n");
8923 }
8924
8925 /**
8926  * intel_init_clock_gating_hooks - setup the clock gating hooks
8927  * @dev_priv: device private
8928  *
8929  * Setup the hooks that configure which clocks of a given platform can be
8930  * gated and also apply various GT and display specific workarounds for these
8931  * platforms. Note that some GT specific workarounds are applied separately
8932  * when GPU contexts or batchbuffers start their execution.
8933  */
8934 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
8935 {
8936         if (IS_CANNONLAKE(dev_priv))
8937                 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
8938         else if (IS_COFFEELAKE(dev_priv))
8939                 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
8940         else if (IS_SKYLAKE(dev_priv))
8941                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
8942         else if (IS_KABYLAKE(dev_priv))
8943                 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
8944         else if (IS_BROXTON(dev_priv))
8945                 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
8946         else if (IS_GEMINILAKE(dev_priv))
8947                 dev_priv->display.init_clock_gating = glk_init_clock_gating;
8948         else if (IS_BROADWELL(dev_priv))
8949                 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
8950         else if (IS_CHERRYVIEW(dev_priv))
8951                 dev_priv->display.init_clock_gating = chv_init_clock_gating;
8952         else if (IS_HASWELL(dev_priv))
8953                 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
8954         else if (IS_IVYBRIDGE(dev_priv))
8955                 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
8956         else if (IS_VALLEYVIEW(dev_priv))
8957                 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
8958         else if (IS_GEN6(dev_priv))
8959                 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
8960         else if (IS_GEN5(dev_priv))
8961                 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
8962         else if (IS_G4X(dev_priv))
8963                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
8964         else if (IS_I965GM(dev_priv))
8965                 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
8966         else if (IS_I965G(dev_priv))
8967                 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
8968         else if (IS_GEN3(dev_priv))
8969                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
8970         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
8971                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
8972         else if (IS_GEN2(dev_priv))
8973                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
8974         else {
8975                 MISSING_CASE(INTEL_DEVID(dev_priv));
8976                 dev_priv->display.init_clock_gating = nop_init_clock_gating;
8977         }
8978 }
8979
8980 /* Set up chip specific power management-related functions */
8981 void intel_init_pm(struct drm_i915_private *dev_priv)
8982 {
8983         intel_fbc_init(dev_priv);
8984
8985         /* For cxsr */
8986         if (IS_PINEVIEW(dev_priv))
8987                 i915_pineview_get_mem_freq(dev_priv);
8988         else if (IS_GEN5(dev_priv))
8989                 i915_ironlake_get_mem_freq(dev_priv);
8990
8991         /* For FIFO watermark updates */
8992         if (INTEL_GEN(dev_priv) >= 9) {
8993                 skl_setup_wm_latency(dev_priv);
8994                 dev_priv->display.initial_watermarks = skl_initial_wm;
8995                 dev_priv->display.atomic_update_watermarks = skl_atomic_update_crtc_wm;
8996                 dev_priv->display.compute_global_watermarks = skl_compute_wm;
8997         } else if (HAS_PCH_SPLIT(dev_priv)) {
8998                 ilk_setup_wm_latency(dev_priv);
8999
9000                 if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] &&
9001                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
9002                     (!IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[0] &&
9003                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
9004                         dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
9005                         dev_priv->display.compute_intermediate_wm =
9006                                 ilk_compute_intermediate_wm;
9007                         dev_priv->display.initial_watermarks =
9008                                 ilk_initial_watermarks;
9009                         dev_priv->display.optimize_watermarks =
9010                                 ilk_optimize_watermarks;
9011                 } else {
9012                         DRM_DEBUG_KMS("Failed to read display plane latency. "
9013                                       "Disable CxSR\n");
9014                 }
9015         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9016                 vlv_setup_wm_latency(dev_priv);
9017                 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
9018                 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
9019                 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
9020                 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
9021                 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
9022         } else if (IS_G4X(dev_priv)) {
9023                 g4x_setup_wm_latency(dev_priv);
9024                 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
9025                 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
9026                 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
9027                 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
9028         } else if (IS_PINEVIEW(dev_priv)) {
9029                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
9030                                             dev_priv->is_ddr3,
9031                                             dev_priv->fsb_freq,
9032                                             dev_priv->mem_freq)) {
9033                         DRM_INFO("failed to find known CxSR latency "
9034                                  "(found ddr%s fsb freq %d, mem freq %d), "
9035                                  "disabling CxSR\n",
9036                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
9037                                  dev_priv->fsb_freq, dev_priv->mem_freq);
9038                         /* Disable CxSR and never update its watermark again */
9039                         intel_set_memory_cxsr(dev_priv, false);
9040                         dev_priv->display.update_wm = NULL;
9041                 } else
9042                         dev_priv->display.update_wm = pineview_update_wm;
9043         } else if (IS_GEN4(dev_priv)) {
9044                 dev_priv->display.update_wm = i965_update_wm;
9045         } else if (IS_GEN3(dev_priv)) {
9046                 dev_priv->display.update_wm = i9xx_update_wm;
9047                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
9048         } else if (IS_GEN2(dev_priv)) {
9049                 if (INTEL_INFO(dev_priv)->num_pipes == 1) {
9050                         dev_priv->display.update_wm = i845_update_wm;
9051                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
9052                 } else {
9053                         dev_priv->display.update_wm = i9xx_update_wm;
9054                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
9055                 }
9056         } else {
9057                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
9058         }
9059 }
9060
9061 static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
9062 {
9063         uint32_t flags =
9064                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9065
9066         switch (flags) {
9067         case GEN6_PCODE_SUCCESS:
9068                 return 0;
9069         case GEN6_PCODE_UNIMPLEMENTED_CMD:
9070                 return -ENODEV;
9071         case GEN6_PCODE_ILLEGAL_CMD:
9072                 return -ENXIO;
9073         case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9074         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9075                 return -EOVERFLOW;
9076         case GEN6_PCODE_TIMEOUT:
9077                 return -ETIMEDOUT;
9078         default:
9079                 MISSING_CASE(flags);
9080                 return 0;
9081         }
9082 }
9083
9084 static inline int gen7_check_mailbox_status(struct drm_i915_private *dev_priv)
9085 {
9086         uint32_t flags =
9087                 I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_ERROR_MASK;
9088
9089         switch (flags) {
9090         case GEN6_PCODE_SUCCESS:
9091                 return 0;
9092         case GEN6_PCODE_ILLEGAL_CMD:
9093                 return -ENXIO;
9094         case GEN7_PCODE_TIMEOUT:
9095                 return -ETIMEDOUT;
9096         case GEN7_PCODE_ILLEGAL_DATA:
9097                 return -EINVAL;
9098         case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
9099                 return -EOVERFLOW;
9100         default:
9101                 MISSING_CASE(flags);
9102                 return 0;
9103         }
9104 }
9105
9106 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
9107 {
9108         int status;
9109
9110         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9111
9112         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9113          * use te fw I915_READ variants to reduce the amount of work
9114          * required when reading/writing.
9115          */
9116
9117         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9118                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps\n",
9119                                  mbox, __builtin_return_address(0));
9120                 return -EAGAIN;
9121         }
9122
9123         I915_WRITE_FW(GEN6_PCODE_DATA, *val);
9124         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9125         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9126
9127         if (__intel_wait_for_register_fw(dev_priv,
9128                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9129                                          500, 0, NULL)) {
9130                 DRM_ERROR("timeout waiting for pcode read (from mbox %x) to finish for %ps\n",
9131                           mbox, __builtin_return_address(0));
9132                 return -ETIMEDOUT;
9133         }
9134
9135         *val = I915_READ_FW(GEN6_PCODE_DATA);
9136         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9137
9138         if (INTEL_GEN(dev_priv) > 6)
9139                 status = gen7_check_mailbox_status(dev_priv);
9140         else
9141                 status = gen6_check_mailbox_status(dev_priv);
9142
9143         if (status) {
9144                 DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n",
9145                                  mbox, __builtin_return_address(0), status);
9146                 return status;
9147         }
9148
9149         return 0;
9150 }
9151
9152 int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv,
9153                                     u32 mbox, u32 val, int timeout_us)
9154 {
9155         int status;
9156
9157         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9158
9159         /* GEN6_PCODE_* are outside of the forcewake domain, we can
9160          * use te fw I915_READ variants to reduce the amount of work
9161          * required when reading/writing.
9162          */
9163
9164         if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
9165                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps\n",
9166                                  val, mbox, __builtin_return_address(0));
9167                 return -EAGAIN;
9168         }
9169
9170         I915_WRITE_FW(GEN6_PCODE_DATA, val);
9171         I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
9172         I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
9173
9174         if (__intel_wait_for_register_fw(dev_priv,
9175                                          GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
9176                                          timeout_us, 0, NULL)) {
9177                 DRM_ERROR("timeout waiting for pcode write of 0x%08x to mbox %x to finish for %ps\n",
9178                           val, mbox, __builtin_return_address(0));
9179                 return -ETIMEDOUT;
9180         }
9181
9182         I915_WRITE_FW(GEN6_PCODE_DATA, 0);
9183
9184         if (INTEL_GEN(dev_priv) > 6)
9185                 status = gen7_check_mailbox_status(dev_priv);
9186         else
9187                 status = gen6_check_mailbox_status(dev_priv);
9188
9189         if (status) {
9190                 DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n",
9191                                  val, mbox, __builtin_return_address(0), status);
9192                 return status;
9193         }
9194
9195         return 0;
9196 }
9197
9198 static bool skl_pcode_try_request(struct drm_i915_private *dev_priv, u32 mbox,
9199                                   u32 request, u32 reply_mask, u32 reply,
9200                                   u32 *status)
9201 {
9202         u32 val = request;
9203
9204         *status = sandybridge_pcode_read(dev_priv, mbox, &val);
9205
9206         return *status || ((val & reply_mask) == reply);
9207 }
9208
9209 /**
9210  * skl_pcode_request - send PCODE request until acknowledgment
9211  * @dev_priv: device private
9212  * @mbox: PCODE mailbox ID the request is targeted for
9213  * @request: request ID
9214  * @reply_mask: mask used to check for request acknowledgment
9215  * @reply: value used to check for request acknowledgment
9216  * @timeout_base_ms: timeout for polling with preemption enabled
9217  *
9218  * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE
9219  * reports an error or an overall timeout of @timeout_base_ms+50 ms expires.
9220  * The request is acknowledged once the PCODE reply dword equals @reply after
9221  * applying @reply_mask. Polling is first attempted with preemption enabled
9222  * for @timeout_base_ms and if this times out for another 50 ms with
9223  * preemption disabled.
9224  *
9225  * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some
9226  * other error as reported by PCODE.
9227  */
9228 int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
9229                       u32 reply_mask, u32 reply, int timeout_base_ms)
9230 {
9231         u32 status;
9232         int ret;
9233
9234         WARN_ON(!mutex_is_locked(&dev_priv->pcu_lock));
9235
9236 #define COND skl_pcode_try_request(dev_priv, mbox, request, reply_mask, reply, \
9237                                    &status)
9238
9239         /*
9240          * Prime the PCODE by doing a request first. Normally it guarantees
9241          * that a subsequent request, at most @timeout_base_ms later, succeeds.
9242          * _wait_for() doesn't guarantee when its passed condition is evaluated
9243          * first, so send the first request explicitly.
9244          */
9245         if (COND) {
9246                 ret = 0;
9247                 goto out;
9248         }
9249         ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10);
9250         if (!ret)
9251                 goto out;
9252
9253         /*
9254          * The above can time out if the number of requests was low (2 in the
9255          * worst case) _and_ PCODE was busy for some reason even after a
9256          * (queued) request and @timeout_base_ms delay. As a workaround retry
9257          * the poll with preemption disabled to maximize the number of
9258          * requests. Increase the timeout from @timeout_base_ms to 50ms to
9259          * account for interrupts that could reduce the number of these
9260          * requests, and for any quirks of the PCODE firmware that delays
9261          * the request completion.
9262          */
9263         DRM_DEBUG_KMS("PCODE timeout, retrying with preemption disabled\n");
9264         WARN_ON_ONCE(timeout_base_ms > 3);
9265         preempt_disable();
9266         ret = wait_for_atomic(COND, 50);
9267         preempt_enable();
9268
9269 out:
9270         return ret ? ret : status;
9271 #undef COND
9272 }
9273
9274 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
9275 {
9276         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9277
9278         /*
9279          * N = val - 0xb7
9280          * Slow = Fast = GPLL ref * N
9281          */
9282         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
9283 }
9284
9285 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
9286 {
9287         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9288
9289         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
9290 }
9291
9292 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
9293 {
9294         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9295
9296         /*
9297          * N = val / 2
9298          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
9299          */
9300         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
9301 }
9302
9303 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
9304 {
9305         struct intel_rps *rps = &dev_priv->gt_pm.rps;
9306
9307         /* CHV needs even values */
9308         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
9309 }
9310
9311 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
9312 {
9313         if (INTEL_GEN(dev_priv) >= 9)
9314                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
9315                                          GEN9_FREQ_SCALER);
9316         else if (IS_CHERRYVIEW(dev_priv))
9317                 return chv_gpu_freq(dev_priv, val);
9318         else if (IS_VALLEYVIEW(dev_priv))
9319                 return byt_gpu_freq(dev_priv, val);
9320         else
9321                 return val * GT_FREQUENCY_MULTIPLIER;
9322 }
9323
9324 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
9325 {
9326         if (INTEL_GEN(dev_priv) >= 9)
9327                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
9328                                          GT_FREQUENCY_MULTIPLIER);
9329         else if (IS_CHERRYVIEW(dev_priv))
9330                 return chv_freq_opcode(dev_priv, val);
9331         else if (IS_VALLEYVIEW(dev_priv))
9332                 return byt_freq_opcode(dev_priv, val);
9333         else
9334                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
9335 }
9336
9337 void intel_pm_setup(struct drm_i915_private *dev_priv)
9338 {
9339         mutex_init(&dev_priv->pcu_lock);
9340
9341         atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
9342
9343         dev_priv->runtime_pm.suspended = false;
9344         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
9345 }
9346
9347 static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
9348                              const i915_reg_t reg)
9349 {
9350         u32 lower, upper, tmp;
9351         unsigned long flags;
9352         int loop = 2;
9353
9354         /* The register accessed do not need forcewake. We borrow
9355          * uncore lock to prevent concurrent access to range reg.
9356          */
9357         spin_lock_irqsave(&dev_priv->uncore.lock, flags);
9358
9359         /* vlv and chv residency counters are 40 bits in width.
9360          * With a control bit, we can choose between upper or lower
9361          * 32bit window into this counter.
9362          *
9363          * Although we always use the counter in high-range mode elsewhere,
9364          * userspace may attempt to read the value before rc6 is initialised,
9365          * before we have set the default VLV_COUNTER_CONTROL value. So always
9366          * set the high bit to be safe.
9367          */
9368         I915_WRITE_FW(VLV_COUNTER_CONTROL,
9369                       _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9370         upper = I915_READ_FW(reg);
9371         do {
9372                 tmp = upper;
9373
9374                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9375                               _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
9376                 lower = I915_READ_FW(reg);
9377
9378                 I915_WRITE_FW(VLV_COUNTER_CONTROL,
9379                               _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
9380                 upper = I915_READ_FW(reg);
9381         } while (upper != tmp && --loop);
9382
9383         /* Everywhere else we always use VLV_COUNTER_CONTROL with the
9384          * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set
9385          * now.
9386          */
9387
9388         spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
9389
9390         return lower | (u64)upper << 8;
9391 }
9392
9393 u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
9394                            const i915_reg_t reg)
9395 {
9396         u64 time_hw;
9397         u32 mul, div;
9398
9399         if (!HAS_RC6(dev_priv))
9400                 return 0;
9401
9402         /* On VLV and CHV, residency time is in CZ units rather than 1.28us */
9403         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
9404                 mul = 1000000;
9405                 div = dev_priv->czclk_freq;
9406                 time_hw = vlv_residency_raw(dev_priv, reg);
9407         } else {
9408                 /* 833.33ns units on Gen9LP, 1.28us elsewhere. */
9409                 if (IS_GEN9_LP(dev_priv)) {
9410                         mul = 10000;
9411                         div = 12;
9412                 } else {
9413                         mul = 1280;
9414                         div = 1;
9415                 }
9416
9417                 time_hw = I915_READ(reg);
9418         }
9419
9420         return DIV_ROUND_UP_ULL(time_hw * mul, div);
9421 }
9422
9423 u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
9424 {
9425         u32 cagf;
9426
9427         if (INTEL_GEN(dev_priv) >= 9)
9428                 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
9429         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
9430                 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
9431         else
9432                 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
9433
9434         return  cagf;
9435 }