2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/module.h>
29 #include <linux/pm_runtime.h>
31 #include <drm/drm_atomic_helper.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_plane_helper.h>
35 #include "display/intel_atomic.h"
36 #include "display/intel_atomic_plane.h"
37 #include "display/intel_bw.h"
38 #include "display/intel_de.h"
39 #include "display/intel_display_trace.h"
40 #include "display/intel_display_types.h"
41 #include "display/intel_fb.h"
42 #include "display/intel_fbc.h"
43 #include "display/intel_sprite.h"
44 #include "display/skl_universal_plane.h"
46 #include "gt/intel_engine_regs.h"
47 #include "gt/intel_gt_regs.h"
48 #include "gt/intel_llc.h"
51 #include "i915_fixed.h"
53 #include "intel_mchbar_regs.h"
54 #include "intel_pcode.h"
56 #include "vlv_sideband.h"
57 #include "../../../platform/x86/intel_ips.h"
59 struct drm_i915_clock_gating_funcs {
60 void (*init_clock_gating)(struct drm_i915_private *i915);
63 /* Stores plane specific WM parameters */
64 struct skl_wm_params {
65 bool x_tiled, y_tiled;
72 u32 plane_bytes_per_line;
73 uint_fixed_16_16_t plane_blocks_per_line;
74 uint_fixed_16_16_t y_tile_minimum;
79 /* used in computing the new watermarks state */
80 struct intel_wm_config {
81 unsigned int num_pipes_active;
86 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
88 if (HAS_LLC(dev_priv)) {
90 * WaCompressedResourceDisplayNewHashMode:skl,kbl
91 * Display WA #0390: skl,kbl
93 * Must match Sampler, Pixel Back End, and Media. See
94 * WaCompressedResourceSamplerPbeMediaNewHashMode.
96 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
97 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
98 SKL_DE_COMPRESSED_HASH_MODE);
101 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
102 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
103 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
105 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
106 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
107 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
110 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
111 * Display WA #0859: skl,bxt,kbl,glk,cfl
113 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
114 DISP_FBC_MEMORY_WAKE);
117 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
119 gen9_init_clock_gating(dev_priv);
121 /* WaDisableSDEUnitClockGating:bxt */
122 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
123 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
127 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
129 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
130 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
133 * Wa: Backlight PWM may stop in the asserted state, causing backlight
136 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
137 PWM1_GATING_DIS | PWM2_GATING_DIS);
140 * Lower the display internal timeout.
141 * This is needed to avoid any hard hangs when DSI port PLL
142 * is off and a MMIO access is attempted by any privilege
143 * application, using batch buffers or any other means.
145 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
148 * WaFbcTurnOffFbcWatermark:bxt
149 * Display WA #0562: bxt
151 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
155 * WaFbcHighMemBwCorruptionAvoidance:bxt
156 * Display WA #0883: bxt
158 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
159 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
160 DPFC_DISABLE_DUMMY0);
163 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
165 gen9_init_clock_gating(dev_priv);
168 * WaDisablePWMClockGating:glk
169 * Backlight PWM may stop in the asserted state, causing backlight
172 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
173 PWM1_GATING_DIS | PWM2_GATING_DIS);
176 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
180 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
182 switch (tmp & CLKCFG_FSB_MASK) {
184 dev_priv->fsb_freq = 533; /* 133*4 */
187 dev_priv->fsb_freq = 800; /* 200*4 */
190 dev_priv->fsb_freq = 667; /* 167*4 */
193 dev_priv->fsb_freq = 400; /* 100*4 */
197 switch (tmp & CLKCFG_MEM_MASK) {
199 dev_priv->mem_freq = 533;
202 dev_priv->mem_freq = 667;
205 dev_priv->mem_freq = 800;
209 /* detect pineview DDR3 setting */
210 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
211 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
214 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
218 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
219 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
221 switch (ddrpll & 0xff) {
223 dev_priv->mem_freq = 800;
226 dev_priv->mem_freq = 1066;
229 dev_priv->mem_freq = 1333;
232 dev_priv->mem_freq = 1600;
235 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
237 dev_priv->mem_freq = 0;
241 switch (csipll & 0x3ff) {
243 dev_priv->fsb_freq = 3200;
246 dev_priv->fsb_freq = 3733;
249 dev_priv->fsb_freq = 4266;
252 dev_priv->fsb_freq = 4800;
255 dev_priv->fsb_freq = 5333;
258 dev_priv->fsb_freq = 5866;
261 dev_priv->fsb_freq = 6400;
264 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
266 dev_priv->fsb_freq = 0;
271 static const struct cxsr_latency cxsr_latency_table[] = {
272 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
273 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
274 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
275 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
276 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
278 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
279 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
280 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
281 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
282 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
284 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
285 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
286 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
287 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
288 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
290 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
291 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
292 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
293 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
294 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
296 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
297 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
298 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
299 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
300 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
302 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
303 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
304 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
305 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
306 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
309 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
314 const struct cxsr_latency *latency;
317 if (fsb == 0 || mem == 0)
320 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
321 latency = &cxsr_latency_table[i];
322 if (is_desktop == latency->is_desktop &&
323 is_ddr3 == latency->is_ddr3 &&
324 fsb == latency->fsb_freq && mem == latency->mem_freq)
328 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
333 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
337 vlv_punit_get(dev_priv);
339 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
341 val &= ~FORCE_DDR_HIGH_FREQ;
343 val |= FORCE_DDR_HIGH_FREQ;
344 val &= ~FORCE_DDR_LOW_FREQ;
345 val |= FORCE_DDR_FREQ_REQ_ACK;
346 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
348 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
349 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
350 drm_err(&dev_priv->drm,
351 "timed out waiting for Punit DDR DVFS request\n");
353 vlv_punit_put(dev_priv);
356 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
360 vlv_punit_get(dev_priv);
362 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
364 val |= DSP_MAXFIFO_PM5_ENABLE;
366 val &= ~DSP_MAXFIFO_PM5_ENABLE;
367 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
369 vlv_punit_put(dev_priv);
372 #define FW_WM(value, plane) \
373 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
375 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
380 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
381 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
382 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
383 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
384 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
385 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
386 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
387 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
388 } else if (IS_PINEVIEW(dev_priv)) {
389 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
390 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
392 val |= PINEVIEW_SELF_REFRESH_EN;
394 val &= ~PINEVIEW_SELF_REFRESH_EN;
395 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
396 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
397 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
398 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
399 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
400 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
401 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
402 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
403 } else if (IS_I915GM(dev_priv)) {
405 * FIXME can't find a bit like this for 915G, and
406 * and yet it does have the related watermark in
407 * FW_BLC_SELF. What's going on?
409 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
410 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
411 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
412 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
413 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
418 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
420 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
421 enableddisabled(enable),
422 enableddisabled(was_enabled));
428 * intel_set_memory_cxsr - Configure CxSR state
429 * @dev_priv: i915 device
430 * @enable: Allow vs. disallow CxSR
432 * Allow or disallow the system to enter a special CxSR
433 * (C-state self refresh) state. What typically happens in CxSR mode
434 * is that several display FIFOs may get combined into a single larger
435 * FIFO for a particular plane (so called max FIFO mode) to allow the
436 * system to defer memory fetches longer, and the memory will enter
439 * Note that enabling CxSR does not guarantee that the system enter
440 * this special mode, nor does it guarantee that the system stays
441 * in that mode once entered. So this just allows/disallows the system
442 * to autonomously utilize the CxSR mode. Other factors such as core
443 * C-states will affect when/if the system actually enters/exits the
446 * Note that on VLV/CHV this actually only controls the max FIFO mode,
447 * and the system is free to enter/exit memory self refresh at any time
448 * even when the use of CxSR has been disallowed.
450 * While the system is actually in the CxSR/max FIFO mode, some plane
451 * control registers will not get latched on vblank. Thus in order to
452 * guarantee the system will respond to changes in the plane registers
453 * we must always disallow CxSR prior to making changes to those registers.
454 * Unfortunately the system will re-evaluate the CxSR conditions at
455 * frame start which happens after vblank start (which is when the plane
456 * registers would get latched), so we can't proceed with the plane update
457 * during the same frame where we disallowed CxSR.
459 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
460 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
461 * the hardware w.r.t. HPLL SR when writing to plane registers.
462 * Disallowing just CxSR is sufficient.
464 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
468 mutex_lock(&dev_priv->wm.wm_mutex);
469 ret = _intel_set_memory_cxsr(dev_priv, enable);
470 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
471 dev_priv->wm.vlv.cxsr = enable;
472 else if (IS_G4X(dev_priv))
473 dev_priv->wm.g4x.cxsr = enable;
474 mutex_unlock(&dev_priv->wm.wm_mutex);
480 * Latency for FIFO fetches is dependent on several factors:
481 * - memory configuration (speed, channels)
483 * - current MCH state
484 * It can be fairly high in some situations, so here we assume a fairly
485 * pessimal value. It's a tradeoff between extra memory fetches (if we
486 * set this value too high, the FIFO will fetch frequently to stay full)
487 * and power consumption (set it too low to save power and we might see
488 * FIFO underruns and display "flicker").
490 * A value of 5us seems to be a good balance; safe for very low end
491 * platforms but not overly aggressive on lower latency configs.
493 static const int pessimal_latency_ns = 5000;
495 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
496 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
498 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
500 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
501 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
502 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
503 enum pipe pipe = crtc->pipe;
504 int sprite0_start, sprite1_start;
505 u32 dsparb, dsparb2, dsparb3;
509 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
510 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
511 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
512 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
515 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
516 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
517 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
518 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
521 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
522 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
523 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
524 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
531 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
532 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
533 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
534 fifo_state->plane[PLANE_CURSOR] = 63;
537 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
538 enum i9xx_plane_id i9xx_plane)
540 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
543 size = dsparb & 0x7f;
544 if (i9xx_plane == PLANE_B)
545 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
547 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
548 dsparb, plane_name(i9xx_plane), size);
553 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
554 enum i9xx_plane_id i9xx_plane)
556 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
559 size = dsparb & 0x1ff;
560 if (i9xx_plane == PLANE_B)
561 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
562 size >>= 1; /* Convert to cachelines */
564 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
565 dsparb, plane_name(i9xx_plane), size);
570 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
571 enum i9xx_plane_id i9xx_plane)
573 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
576 size = dsparb & 0x7f;
577 size >>= 2; /* Convert to cachelines */
579 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
580 dsparb, plane_name(i9xx_plane), size);
585 /* Pineview has different values for various configs */
586 static const struct intel_watermark_params pnv_display_wm = {
587 .fifo_size = PINEVIEW_DISPLAY_FIFO,
588 .max_wm = PINEVIEW_MAX_WM,
589 .default_wm = PINEVIEW_DFT_WM,
590 .guard_size = PINEVIEW_GUARD_WM,
591 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
594 static const struct intel_watermark_params pnv_display_hplloff_wm = {
595 .fifo_size = PINEVIEW_DISPLAY_FIFO,
596 .max_wm = PINEVIEW_MAX_WM,
597 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
598 .guard_size = PINEVIEW_GUARD_WM,
599 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
602 static const struct intel_watermark_params pnv_cursor_wm = {
603 .fifo_size = PINEVIEW_CURSOR_FIFO,
604 .max_wm = PINEVIEW_CURSOR_MAX_WM,
605 .default_wm = PINEVIEW_CURSOR_DFT_WM,
606 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
607 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
610 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
611 .fifo_size = PINEVIEW_CURSOR_FIFO,
612 .max_wm = PINEVIEW_CURSOR_MAX_WM,
613 .default_wm = PINEVIEW_CURSOR_DFT_WM,
614 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
615 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
618 static const struct intel_watermark_params i965_cursor_wm_info = {
619 .fifo_size = I965_CURSOR_FIFO,
620 .max_wm = I965_CURSOR_MAX_WM,
621 .default_wm = I965_CURSOR_DFT_WM,
623 .cacheline_size = I915_FIFO_LINE_SIZE,
626 static const struct intel_watermark_params i945_wm_info = {
627 .fifo_size = I945_FIFO_SIZE,
628 .max_wm = I915_MAX_WM,
631 .cacheline_size = I915_FIFO_LINE_SIZE,
634 static const struct intel_watermark_params i915_wm_info = {
635 .fifo_size = I915_FIFO_SIZE,
636 .max_wm = I915_MAX_WM,
639 .cacheline_size = I915_FIFO_LINE_SIZE,
642 static const struct intel_watermark_params i830_a_wm_info = {
643 .fifo_size = I855GM_FIFO_SIZE,
644 .max_wm = I915_MAX_WM,
647 .cacheline_size = I830_FIFO_LINE_SIZE,
650 static const struct intel_watermark_params i830_bc_wm_info = {
651 .fifo_size = I855GM_FIFO_SIZE,
652 .max_wm = I915_MAX_WM/2,
655 .cacheline_size = I830_FIFO_LINE_SIZE,
658 static const struct intel_watermark_params i845_wm_info = {
659 .fifo_size = I830_FIFO_SIZE,
660 .max_wm = I915_MAX_WM,
663 .cacheline_size = I830_FIFO_LINE_SIZE,
667 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
668 * @pixel_rate: Pipe pixel rate in kHz
669 * @cpp: Plane bytes per pixel
670 * @latency: Memory wakeup latency in 0.1us units
672 * Compute the watermark using the method 1 or "small buffer"
673 * formula. The caller may additonally add extra cachelines
674 * to account for TLB misses and clock crossings.
676 * This method is concerned with the short term drain rate
677 * of the FIFO, ie. it does not account for blanking periods
678 * which would effectively reduce the average drain rate across
679 * a longer period. The name "small" refers to the fact the
680 * FIFO is relatively small compared to the amount of data
683 * The FIFO level vs. time graph might look something like:
687 * __---__---__ (- plane active, _ blanking)
690 * or perhaps like this:
693 * __----__----__ (- plane active, _ blanking)
697 * The watermark in bytes
699 static unsigned int intel_wm_method1(unsigned int pixel_rate,
701 unsigned int latency)
705 ret = mul_u32_u32(pixel_rate, cpp * latency);
706 ret = DIV_ROUND_UP_ULL(ret, 10000);
712 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
713 * @pixel_rate: Pipe pixel rate in kHz
714 * @htotal: Pipe horizontal total
715 * @width: Plane width in pixels
716 * @cpp: Plane bytes per pixel
717 * @latency: Memory wakeup latency in 0.1us units
719 * Compute the watermark using the method 2 or "large buffer"
720 * formula. The caller may additonally add extra cachelines
721 * to account for TLB misses and clock crossings.
723 * This method is concerned with the long term drain rate
724 * of the FIFO, ie. it does account for blanking periods
725 * which effectively reduce the average drain rate across
726 * a longer period. The name "large" refers to the fact the
727 * FIFO is relatively large compared to the amount of data
730 * The FIFO level vs. time graph might look something like:
735 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
739 * The watermark in bytes
741 static unsigned int intel_wm_method2(unsigned int pixel_rate,
745 unsigned int latency)
750 * FIXME remove once all users are computing
751 * watermarks in the correct place.
753 if (WARN_ON_ONCE(htotal == 0))
756 ret = (latency * pixel_rate) / (htotal * 10000);
757 ret = (ret + 1) * width * cpp;
763 * intel_calculate_wm - calculate watermark level
764 * @pixel_rate: pixel clock
765 * @wm: chip FIFO params
766 * @fifo_size: size of the FIFO buffer
767 * @cpp: bytes per pixel
768 * @latency_ns: memory latency for the platform
770 * Calculate the watermark level (the level at which the display plane will
771 * start fetching from memory again). Each chip has a different display
772 * FIFO size and allocation, so the caller needs to figure that out and pass
773 * in the correct intel_watermark_params structure.
775 * As the pixel clock runs, the FIFO will be drained at a rate that depends
776 * on the pixel size. When it reaches the watermark level, it'll start
777 * fetching FIFO line sized based chunks from memory until the FIFO fills
778 * past the watermark point. If the FIFO drains completely, a FIFO underrun
779 * will occur, and a display engine hang could result.
781 static unsigned int intel_calculate_wm(int pixel_rate,
782 const struct intel_watermark_params *wm,
783 int fifo_size, int cpp,
784 unsigned int latency_ns)
786 int entries, wm_size;
789 * Note: we need to make sure we don't overflow for various clock &
791 * clocks go from a few thousand to several hundred thousand.
792 * latency is usually a few thousand
794 entries = intel_wm_method1(pixel_rate, cpp,
796 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
798 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
800 wm_size = fifo_size - entries;
801 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
803 /* Don't promote wm_size to unsigned... */
804 if (wm_size > wm->max_wm)
805 wm_size = wm->max_wm;
807 wm_size = wm->default_wm;
810 * Bspec seems to indicate that the value shouldn't be lower than
811 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
812 * Lets go for 8 which is the burst size since certain platforms
813 * already use a hardcoded 8 (which is what the spec says should be
822 static bool is_disabling(int old, int new, int threshold)
824 return old >= threshold && new < threshold;
827 static bool is_enabling(int old, int new, int threshold)
829 return old < threshold && new >= threshold;
832 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
834 return dev_priv->wm.max_level + 1;
837 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
838 const struct intel_plane_state *plane_state)
840 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
842 /* FIXME check the 'enable' instead */
843 if (!crtc_state->hw.active)
847 * Treat cursor with fb as always visible since cursor updates
848 * can happen faster than the vrefresh rate, and the current
849 * watermark code doesn't handle that correctly. Cursor updates
850 * which set/clear the fb or change the cursor size are going
851 * to get throttled by intel_legacy_cursor_update() to work
852 * around this problem with the watermark code.
854 if (plane->id == PLANE_CURSOR)
855 return plane_state->hw.fb != NULL;
857 return plane_state->uapi.visible;
860 static bool intel_crtc_active(struct intel_crtc *crtc)
862 /* Be paranoid as we can arrive here with only partial
863 * state retrieved from the hardware during setup.
865 * We can ditch the adjusted_mode.crtc_clock check as soon
866 * as Haswell has gained clock readout/fastboot support.
868 * We can ditch the crtc->primary->state->fb check as soon as we can
869 * properly reconstruct framebuffers.
871 * FIXME: The intel_crtc->active here should be switched to
872 * crtc->state->active once we have proper CRTC states wired up
875 return crtc && crtc->active && crtc->base.primary->state->fb &&
876 crtc->config->hw.adjusted_mode.crtc_clock;
879 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
881 struct intel_crtc *crtc, *enabled = NULL;
883 for_each_intel_crtc(&dev_priv->drm, crtc) {
884 if (intel_crtc_active(crtc)) {
894 static void pnv_update_wm(struct drm_i915_private *dev_priv)
896 struct intel_crtc *crtc;
897 const struct cxsr_latency *latency;
901 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
906 drm_dbg_kms(&dev_priv->drm,
907 "Unknown FSB/MEM found, disable CxSR\n");
908 intel_set_memory_cxsr(dev_priv, false);
912 crtc = single_enabled_crtc(dev_priv);
914 const struct drm_framebuffer *fb =
915 crtc->base.primary->state->fb;
916 int pixel_rate = crtc->config->pixel_rate;
917 int cpp = fb->format->cpp[0];
920 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
921 pnv_display_wm.fifo_size,
922 cpp, latency->display_sr);
923 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
924 reg &= ~DSPFW_SR_MASK;
925 reg |= FW_WM(wm, SR);
926 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
927 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
930 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
931 pnv_display_wm.fifo_size,
932 4, latency->cursor_sr);
933 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
934 reg &= ~DSPFW_CURSOR_SR_MASK;
935 reg |= FW_WM(wm, CURSOR_SR);
936 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
938 /* Display HPLL off SR */
939 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
940 pnv_display_hplloff_wm.fifo_size,
941 cpp, latency->display_hpll_disable);
942 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
943 reg &= ~DSPFW_HPLL_SR_MASK;
944 reg |= FW_WM(wm, HPLL_SR);
945 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
947 /* cursor HPLL off SR */
948 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
949 pnv_display_hplloff_wm.fifo_size,
950 4, latency->cursor_hpll_disable);
951 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
952 reg &= ~DSPFW_HPLL_CURSOR_MASK;
953 reg |= FW_WM(wm, HPLL_CURSOR);
954 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
955 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
957 intel_set_memory_cxsr(dev_priv, true);
959 intel_set_memory_cxsr(dev_priv, false);
964 * Documentation says:
965 * "If the line size is small, the TLB fetches can get in the way of the
966 * data fetches, causing some lag in the pixel data return which is not
967 * accounted for in the above formulas. The following adjustment only
968 * needs to be applied if eight whole lines fit in the buffer at once.
969 * The WM is adjusted upwards by the difference between the FIFO size
970 * and the size of 8 whole lines. This adjustment is always performed
971 * in the actual pixel depth regardless of whether FBC is enabled or not."
973 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
975 int tlb_miss = fifo_size * 64 - width * cpp * 8;
977 return max(0, tlb_miss);
980 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
981 const struct g4x_wm_values *wm)
985 for_each_pipe(dev_priv, pipe)
986 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
988 intel_uncore_write(&dev_priv->uncore, DSPFW1,
989 FW_WM(wm->sr.plane, SR) |
990 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
991 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
992 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
993 intel_uncore_write(&dev_priv->uncore, DSPFW2,
994 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
995 FW_WM(wm->sr.fbc, FBC_SR) |
996 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
997 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
998 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
999 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1000 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1001 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1002 FW_WM(wm->sr.cursor, CURSOR_SR) |
1003 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1004 FW_WM(wm->hpll.plane, HPLL_SR));
1006 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1009 #define FW_WM_VLV(value, plane) \
1010 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1012 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1013 const struct vlv_wm_values *wm)
1017 for_each_pipe(dev_priv, pipe) {
1018 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
1020 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1021 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1022 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1023 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1024 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1028 * Zero the (unused) WM1 watermarks, and also clear all the
1029 * high order bits so that there are no out of bounds values
1030 * present in the registers during the reprogramming.
1032 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1033 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1034 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1035 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1036 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1038 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1039 FW_WM(wm->sr.plane, SR) |
1040 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1041 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1042 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1043 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1044 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1045 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1046 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1047 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1048 FW_WM(wm->sr.cursor, CURSOR_SR));
1050 if (IS_CHERRYVIEW(dev_priv)) {
1051 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1052 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1053 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1054 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1055 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1056 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1057 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1058 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1059 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1060 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1061 FW_WM(wm->sr.plane >> 9, SR_HI) |
1062 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1063 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1064 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1065 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1066 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1067 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1068 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1069 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1070 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1072 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1073 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1074 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1075 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1076 FW_WM(wm->sr.plane >> 9, SR_HI) |
1077 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1078 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1079 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1080 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1081 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1082 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1085 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1090 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1092 /* all latencies in usec */
1093 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1094 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1095 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1097 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1100 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1103 * DSPCNTR[13] supposedly controls whether the
1104 * primary plane can use the FIFO space otherwise
1105 * reserved for the sprite plane. It's not 100% clear
1106 * what the actual FIFO size is, but it looks like we
1107 * can happily set both primary and sprite watermarks
1108 * up to 127 cachelines. So that would seem to mean
1109 * that either DSPCNTR[13] doesn't do anything, or that
1110 * the total FIFO is >= 256 cachelines in size. Either
1111 * way, we don't seem to have to worry about this
1112 * repartitioning as the maximum watermark value the
1113 * register can hold for each plane is lower than the
1114 * minimum FIFO size.
1120 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1122 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1124 MISSING_CASE(plane_id);
1129 static int g4x_fbc_fifo_size(int level)
1132 case G4X_WM_LEVEL_SR:
1134 case G4X_WM_LEVEL_HPLL:
1137 MISSING_CASE(level);
1142 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1143 const struct intel_plane_state *plane_state,
1146 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1147 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1148 const struct drm_display_mode *pipe_mode =
1149 &crtc_state->hw.pipe_mode;
1150 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1151 unsigned int pixel_rate, htotal, cpp, width, wm;
1156 if (!intel_wm_plane_visible(crtc_state, plane_state))
1159 cpp = plane_state->hw.fb->format->cpp[0];
1162 * WaUse32BppForSRWM:ctg,elk
1164 * The spec fails to list this restriction for the
1165 * HPLL watermark, which seems a little strange.
1166 * Let's use 32bpp for the HPLL watermark as well.
1168 if (plane->id == PLANE_PRIMARY &&
1169 level != G4X_WM_LEVEL_NORMAL)
1172 pixel_rate = crtc_state->pixel_rate;
1173 htotal = pipe_mode->crtc_htotal;
1174 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1176 if (plane->id == PLANE_CURSOR) {
1177 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1178 } else if (plane->id == PLANE_PRIMARY &&
1179 level == G4X_WM_LEVEL_NORMAL) {
1180 wm = intel_wm_method1(pixel_rate, cpp, latency);
1182 unsigned int small, large;
1184 small = intel_wm_method1(pixel_rate, cpp, latency);
1185 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1187 wm = min(small, large);
1190 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1193 wm = DIV_ROUND_UP(wm, 64) + 2;
1195 return min_t(unsigned int, wm, USHRT_MAX);
1198 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1199 int level, enum plane_id plane_id, u16 value)
1201 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1204 for (; level < intel_wm_num_levels(dev_priv); level++) {
1205 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1207 dirty |= raw->plane[plane_id] != value;
1208 raw->plane[plane_id] = value;
1214 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1215 int level, u16 value)
1217 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1220 /* NORMAL level doesn't have an FBC watermark */
1221 level = max(level, G4X_WM_LEVEL_SR);
1223 for (; level < intel_wm_num_levels(dev_priv); level++) {
1224 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1226 dirty |= raw->fbc != value;
1233 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1234 const struct intel_plane_state *plane_state,
1237 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1238 const struct intel_plane_state *plane_state)
1240 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1241 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1242 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1243 enum plane_id plane_id = plane->id;
1247 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1248 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1249 if (plane_id == PLANE_PRIMARY)
1250 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1254 for (level = 0; level < num_levels; level++) {
1255 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1258 wm = g4x_compute_wm(crtc_state, plane_state, level);
1259 max_wm = g4x_plane_fifo_size(plane_id, level);
1264 dirty |= raw->plane[plane_id] != wm;
1265 raw->plane[plane_id] = wm;
1267 if (plane_id != PLANE_PRIMARY ||
1268 level == G4X_WM_LEVEL_NORMAL)
1271 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1272 raw->plane[plane_id]);
1273 max_wm = g4x_fbc_fifo_size(level);
1276 * FBC wm is not mandatory as we
1277 * can always just disable its use.
1282 dirty |= raw->fbc != wm;
1286 /* mark watermarks as invalid */
1287 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1289 if (plane_id == PLANE_PRIMARY)
1290 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1294 drm_dbg_kms(&dev_priv->drm,
1295 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1297 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1298 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1299 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1301 if (plane_id == PLANE_PRIMARY)
1302 drm_dbg_kms(&dev_priv->drm,
1303 "FBC watermarks: SR=%d, HPLL=%d\n",
1304 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1305 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1311 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1312 enum plane_id plane_id, int level)
1314 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1316 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1319 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1322 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1324 if (level > dev_priv->wm.max_level)
1327 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1328 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1329 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1332 /* mark all levels starting from 'level' as invalid */
1333 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1334 struct g4x_wm_state *wm_state, int level)
1336 if (level <= G4X_WM_LEVEL_NORMAL) {
1337 enum plane_id plane_id;
1339 for_each_plane_id_on_crtc(crtc, plane_id)
1340 wm_state->wm.plane[plane_id] = USHRT_MAX;
1343 if (level <= G4X_WM_LEVEL_SR) {
1344 wm_state->cxsr = false;
1345 wm_state->sr.cursor = USHRT_MAX;
1346 wm_state->sr.plane = USHRT_MAX;
1347 wm_state->sr.fbc = USHRT_MAX;
1350 if (level <= G4X_WM_LEVEL_HPLL) {
1351 wm_state->hpll_en = false;
1352 wm_state->hpll.cursor = USHRT_MAX;
1353 wm_state->hpll.plane = USHRT_MAX;
1354 wm_state->hpll.fbc = USHRT_MAX;
1358 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1361 if (level < G4X_WM_LEVEL_SR)
1364 if (level >= G4X_WM_LEVEL_SR &&
1365 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1368 if (level >= G4X_WM_LEVEL_HPLL &&
1369 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1375 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1376 struct intel_crtc *crtc)
1378 struct intel_crtc_state *crtc_state =
1379 intel_atomic_get_new_crtc_state(state, crtc);
1380 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1381 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1382 const struct g4x_pipe_wm *raw;
1383 const struct intel_plane_state *old_plane_state;
1384 const struct intel_plane_state *new_plane_state;
1385 struct intel_plane *plane;
1386 enum plane_id plane_id;
1388 unsigned int dirty = 0;
1390 for_each_oldnew_intel_plane_in_state(state, plane,
1392 new_plane_state, i) {
1393 if (new_plane_state->hw.crtc != &crtc->base &&
1394 old_plane_state->hw.crtc != &crtc->base)
1397 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1398 dirty |= BIT(plane->id);
1404 level = G4X_WM_LEVEL_NORMAL;
1405 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1408 raw = &crtc_state->wm.g4x.raw[level];
1409 for_each_plane_id_on_crtc(crtc, plane_id)
1410 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1412 level = G4X_WM_LEVEL_SR;
1413 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1416 raw = &crtc_state->wm.g4x.raw[level];
1417 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1418 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1419 wm_state->sr.fbc = raw->fbc;
1421 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1423 level = G4X_WM_LEVEL_HPLL;
1424 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1427 raw = &crtc_state->wm.g4x.raw[level];
1428 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1429 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1430 wm_state->hpll.fbc = raw->fbc;
1432 wm_state->hpll_en = wm_state->cxsr;
1437 if (level == G4X_WM_LEVEL_NORMAL)
1440 /* invalidate the higher levels */
1441 g4x_invalidate_wms(crtc, wm_state, level);
1444 * Determine if the FBC watermark(s) can be used. IF
1445 * this isn't the case we prefer to disable the FBC
1446 * watermark(s) rather than disable the SR/HPLL
1447 * level(s) entirely. 'level-1' is the highest valid
1450 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1455 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1456 struct intel_crtc *crtc)
1458 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1459 struct intel_crtc_state *new_crtc_state =
1460 intel_atomic_get_new_crtc_state(state, crtc);
1461 const struct intel_crtc_state *old_crtc_state =
1462 intel_atomic_get_old_crtc_state(state, crtc);
1463 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1464 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1465 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1466 enum plane_id plane_id;
1468 if (!new_crtc_state->hw.active ||
1469 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1470 *intermediate = *optimal;
1472 intermediate->cxsr = false;
1473 intermediate->hpll_en = false;
1477 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1478 !new_crtc_state->disable_cxsr;
1479 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1480 !new_crtc_state->disable_cxsr;
1481 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1483 for_each_plane_id_on_crtc(crtc, plane_id) {
1484 intermediate->wm.plane[plane_id] =
1485 max(optimal->wm.plane[plane_id],
1486 active->wm.plane[plane_id]);
1488 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1489 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1492 intermediate->sr.plane = max(optimal->sr.plane,
1494 intermediate->sr.cursor = max(optimal->sr.cursor,
1496 intermediate->sr.fbc = max(optimal->sr.fbc,
1499 intermediate->hpll.plane = max(optimal->hpll.plane,
1500 active->hpll.plane);
1501 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1502 active->hpll.cursor);
1503 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1506 drm_WARN_ON(&dev_priv->drm,
1507 (intermediate->sr.plane >
1508 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1509 intermediate->sr.cursor >
1510 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1511 intermediate->cxsr);
1512 drm_WARN_ON(&dev_priv->drm,
1513 (intermediate->sr.plane >
1514 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1515 intermediate->sr.cursor >
1516 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1517 intermediate->hpll_en);
1519 drm_WARN_ON(&dev_priv->drm,
1520 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1521 intermediate->fbc_en && intermediate->cxsr);
1522 drm_WARN_ON(&dev_priv->drm,
1523 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1524 intermediate->fbc_en && intermediate->hpll_en);
1528 * If our intermediate WM are identical to the final WM, then we can
1529 * omit the post-vblank programming; only update if it's different.
1531 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1532 new_crtc_state->wm.need_postvbl_update = true;
1537 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1538 struct g4x_wm_values *wm)
1540 struct intel_crtc *crtc;
1541 int num_active_pipes = 0;
1547 for_each_intel_crtc(&dev_priv->drm, crtc) {
1548 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1553 if (!wm_state->cxsr)
1555 if (!wm_state->hpll_en)
1556 wm->hpll_en = false;
1557 if (!wm_state->fbc_en)
1563 if (num_active_pipes != 1) {
1565 wm->hpll_en = false;
1569 for_each_intel_crtc(&dev_priv->drm, crtc) {
1570 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1571 enum pipe pipe = crtc->pipe;
1573 wm->pipe[pipe] = wm_state->wm;
1574 if (crtc->active && wm->cxsr)
1575 wm->sr = wm_state->sr;
1576 if (crtc->active && wm->hpll_en)
1577 wm->hpll = wm_state->hpll;
1581 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1583 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1584 struct g4x_wm_values new_wm = {};
1586 g4x_merge_wm(dev_priv, &new_wm);
1588 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1591 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1592 _intel_set_memory_cxsr(dev_priv, false);
1594 g4x_write_wm_values(dev_priv, &new_wm);
1596 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1597 _intel_set_memory_cxsr(dev_priv, true);
1602 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1603 struct intel_crtc *crtc)
1605 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1606 const struct intel_crtc_state *crtc_state =
1607 intel_atomic_get_new_crtc_state(state, crtc);
1609 mutex_lock(&dev_priv->wm.wm_mutex);
1610 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1611 g4x_program_watermarks(dev_priv);
1612 mutex_unlock(&dev_priv->wm.wm_mutex);
1615 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1616 struct intel_crtc *crtc)
1618 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1619 const struct intel_crtc_state *crtc_state =
1620 intel_atomic_get_new_crtc_state(state, crtc);
1622 if (!crtc_state->wm.need_postvbl_update)
1625 mutex_lock(&dev_priv->wm.wm_mutex);
1626 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1627 g4x_program_watermarks(dev_priv);
1628 mutex_unlock(&dev_priv->wm.wm_mutex);
1631 /* latency must be in 0.1us units. */
1632 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1633 unsigned int htotal,
1636 unsigned int latency)
1640 ret = intel_wm_method2(pixel_rate, htotal,
1641 width, cpp, latency);
1642 ret = DIV_ROUND_UP(ret, 64);
1647 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1649 /* all latencies in usec */
1650 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1652 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1654 if (IS_CHERRYVIEW(dev_priv)) {
1655 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1656 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1658 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1662 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1663 const struct intel_plane_state *plane_state,
1666 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1667 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1668 const struct drm_display_mode *pipe_mode =
1669 &crtc_state->hw.pipe_mode;
1670 unsigned int pixel_rate, htotal, cpp, width, wm;
1672 if (dev_priv->wm.pri_latency[level] == 0)
1675 if (!intel_wm_plane_visible(crtc_state, plane_state))
1678 cpp = plane_state->hw.fb->format->cpp[0];
1679 pixel_rate = crtc_state->pixel_rate;
1680 htotal = pipe_mode->crtc_htotal;
1681 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1683 if (plane->id == PLANE_CURSOR) {
1685 * FIXME the formula gives values that are
1686 * too big for the cursor FIFO, and hence we
1687 * would never be able to use cursors. For
1688 * now just hardcode the watermark.
1692 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1693 dev_priv->wm.pri_latency[level] * 10);
1696 return min_t(unsigned int, wm, USHRT_MAX);
1699 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1701 return (active_planes & (BIT(PLANE_SPRITE0) |
1702 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1705 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1707 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1708 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1709 const struct g4x_pipe_wm *raw =
1710 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1711 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1712 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1713 int num_active_planes = hweight8(active_planes);
1714 const int fifo_size = 511;
1715 int fifo_extra, fifo_left = fifo_size;
1716 int sprite0_fifo_extra = 0;
1717 unsigned int total_rate;
1718 enum plane_id plane_id;
1721 * When enabling sprite0 after sprite1 has already been enabled
1722 * we tend to get an underrun unless sprite0 already has some
1723 * FIFO space allcoated. Hence we always allocate at least one
1724 * cacheline for sprite0 whenever sprite1 is enabled.
1726 * All other plane enable sequences appear immune to this problem.
1728 if (vlv_need_sprite0_fifo_workaround(active_planes))
1729 sprite0_fifo_extra = 1;
1731 total_rate = raw->plane[PLANE_PRIMARY] +
1732 raw->plane[PLANE_SPRITE0] +
1733 raw->plane[PLANE_SPRITE1] +
1736 if (total_rate > fifo_size)
1739 if (total_rate == 0)
1742 for_each_plane_id_on_crtc(crtc, plane_id) {
1745 if ((active_planes & BIT(plane_id)) == 0) {
1746 fifo_state->plane[plane_id] = 0;
1750 rate = raw->plane[plane_id];
1751 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1752 fifo_left -= fifo_state->plane[plane_id];
1755 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1756 fifo_left -= sprite0_fifo_extra;
1758 fifo_state->plane[PLANE_CURSOR] = 63;
1760 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1762 /* spread the remainder evenly */
1763 for_each_plane_id_on_crtc(crtc, plane_id) {
1769 if ((active_planes & BIT(plane_id)) == 0)
1772 plane_extra = min(fifo_extra, fifo_left);
1773 fifo_state->plane[plane_id] += plane_extra;
1774 fifo_left -= plane_extra;
1777 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1779 /* give it all to the first plane if none are active */
1780 if (active_planes == 0) {
1781 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1782 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1788 /* mark all levels starting from 'level' as invalid */
1789 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1790 struct vlv_wm_state *wm_state, int level)
1792 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1794 for (; level < intel_wm_num_levels(dev_priv); level++) {
1795 enum plane_id plane_id;
1797 for_each_plane_id_on_crtc(crtc, plane_id)
1798 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1800 wm_state->sr[level].cursor = USHRT_MAX;
1801 wm_state->sr[level].plane = USHRT_MAX;
1805 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1810 return fifo_size - wm;
1814 * Starting from 'level' set all higher
1815 * levels to 'value' in the "raw" watermarks.
1817 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1818 int level, enum plane_id plane_id, u16 value)
1820 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1821 int num_levels = intel_wm_num_levels(dev_priv);
1824 for (; level < num_levels; level++) {
1825 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1827 dirty |= raw->plane[plane_id] != value;
1828 raw->plane[plane_id] = value;
1834 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1835 const struct intel_plane_state *plane_state)
1837 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1838 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1839 enum plane_id plane_id = plane->id;
1840 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1844 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1845 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1849 for (level = 0; level < num_levels; level++) {
1850 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1851 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1852 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1857 dirty |= raw->plane[plane_id] != wm;
1858 raw->plane[plane_id] = wm;
1861 /* mark all higher levels as invalid */
1862 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1866 drm_dbg_kms(&dev_priv->drm,
1867 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1869 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1870 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1871 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1876 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1877 enum plane_id plane_id, int level)
1879 const struct g4x_pipe_wm *raw =
1880 &crtc_state->wm.vlv.raw[level];
1881 const struct vlv_fifo_state *fifo_state =
1882 &crtc_state->wm.vlv.fifo_state;
1884 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1887 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1889 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1890 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1891 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1892 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1895 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1896 struct intel_crtc *crtc)
1898 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1899 struct intel_crtc_state *crtc_state =
1900 intel_atomic_get_new_crtc_state(state, crtc);
1901 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1902 const struct vlv_fifo_state *fifo_state =
1903 &crtc_state->wm.vlv.fifo_state;
1904 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1905 int num_active_planes = hweight8(active_planes);
1906 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1907 const struct intel_plane_state *old_plane_state;
1908 const struct intel_plane_state *new_plane_state;
1909 struct intel_plane *plane;
1910 enum plane_id plane_id;
1912 unsigned int dirty = 0;
1914 for_each_oldnew_intel_plane_in_state(state, plane,
1916 new_plane_state, i) {
1917 if (new_plane_state->hw.crtc != &crtc->base &&
1918 old_plane_state->hw.crtc != &crtc->base)
1921 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1922 dirty |= BIT(plane->id);
1926 * DSPARB registers may have been reset due to the
1927 * power well being turned off. Make sure we restore
1928 * them to a consistent state even if no primary/sprite
1929 * planes are initially active.
1932 crtc_state->fifo_changed = true;
1937 /* cursor changes don't warrant a FIFO recompute */
1938 if (dirty & ~BIT(PLANE_CURSOR)) {
1939 const struct intel_crtc_state *old_crtc_state =
1940 intel_atomic_get_old_crtc_state(state, crtc);
1941 const struct vlv_fifo_state *old_fifo_state =
1942 &old_crtc_state->wm.vlv.fifo_state;
1944 ret = vlv_compute_fifo(crtc_state);
1948 if (needs_modeset ||
1949 memcmp(old_fifo_state, fifo_state,
1950 sizeof(*fifo_state)) != 0)
1951 crtc_state->fifo_changed = true;
1954 /* initially allow all levels */
1955 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1957 * Note that enabling cxsr with no primary/sprite planes
1958 * enabled can wedge the pipe. Hence we only allow cxsr
1959 * with exactly one enabled primary/sprite plane.
1961 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1963 for (level = 0; level < wm_state->num_levels; level++) {
1964 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1965 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1967 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1970 for_each_plane_id_on_crtc(crtc, plane_id) {
1971 wm_state->wm[level].plane[plane_id] =
1972 vlv_invert_wm_value(raw->plane[plane_id],
1973 fifo_state->plane[plane_id]);
1976 wm_state->sr[level].plane =
1977 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1978 raw->plane[PLANE_SPRITE0],
1979 raw->plane[PLANE_SPRITE1]),
1982 wm_state->sr[level].cursor =
1983 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1990 /* limit to only levels we can actually handle */
1991 wm_state->num_levels = level;
1993 /* invalidate the higher levels */
1994 vlv_invalidate_wms(crtc, wm_state, level);
1999 #define VLV_FIFO(plane, value) \
2000 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2002 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2003 struct intel_crtc *crtc)
2005 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2006 struct intel_uncore *uncore = &dev_priv->uncore;
2007 const struct intel_crtc_state *crtc_state =
2008 intel_atomic_get_new_crtc_state(state, crtc);
2009 const struct vlv_fifo_state *fifo_state =
2010 &crtc_state->wm.vlv.fifo_state;
2011 int sprite0_start, sprite1_start, fifo_size;
2012 u32 dsparb, dsparb2, dsparb3;
2014 if (!crtc_state->fifo_changed)
2017 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2018 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2019 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2021 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2022 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2024 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2027 * uncore.lock serves a double purpose here. It allows us to
2028 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2029 * it protects the DSPARB registers from getting clobbered by
2030 * parallel updates from multiple pipes.
2032 * intel_pipe_update_start() has already disabled interrupts
2033 * for us, so a plain spin_lock() is sufficient here.
2035 spin_lock(&uncore->lock);
2037 switch (crtc->pipe) {
2039 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2040 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2042 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2043 VLV_FIFO(SPRITEB, 0xff));
2044 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2045 VLV_FIFO(SPRITEB, sprite1_start));
2047 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2048 VLV_FIFO(SPRITEB_HI, 0x1));
2049 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2050 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2052 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2053 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2056 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2057 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2059 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2060 VLV_FIFO(SPRITED, 0xff));
2061 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2062 VLV_FIFO(SPRITED, sprite1_start));
2064 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2065 VLV_FIFO(SPRITED_HI, 0xff));
2066 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2067 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2069 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2070 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2073 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2074 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2076 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2077 VLV_FIFO(SPRITEF, 0xff));
2078 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2079 VLV_FIFO(SPRITEF, sprite1_start));
2081 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2082 VLV_FIFO(SPRITEF_HI, 0xff));
2083 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2084 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2086 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2087 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2093 intel_uncore_posting_read_fw(uncore, DSPARB);
2095 spin_unlock(&uncore->lock);
2100 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2101 struct intel_crtc *crtc)
2103 struct intel_crtc_state *new_crtc_state =
2104 intel_atomic_get_new_crtc_state(state, crtc);
2105 const struct intel_crtc_state *old_crtc_state =
2106 intel_atomic_get_old_crtc_state(state, crtc);
2107 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2108 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2109 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2112 if (!new_crtc_state->hw.active ||
2113 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2114 *intermediate = *optimal;
2116 intermediate->cxsr = false;
2120 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2121 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2122 !new_crtc_state->disable_cxsr;
2124 for (level = 0; level < intermediate->num_levels; level++) {
2125 enum plane_id plane_id;
2127 for_each_plane_id_on_crtc(crtc, plane_id) {
2128 intermediate->wm[level].plane[plane_id] =
2129 min(optimal->wm[level].plane[plane_id],
2130 active->wm[level].plane[plane_id]);
2133 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2134 active->sr[level].plane);
2135 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2136 active->sr[level].cursor);
2139 vlv_invalidate_wms(crtc, intermediate, level);
2143 * If our intermediate WM are identical to the final WM, then we can
2144 * omit the post-vblank programming; only update if it's different.
2146 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2147 new_crtc_state->wm.need_postvbl_update = true;
2152 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2153 struct vlv_wm_values *wm)
2155 struct intel_crtc *crtc;
2156 int num_active_pipes = 0;
2158 wm->level = dev_priv->wm.max_level;
2161 for_each_intel_crtc(&dev_priv->drm, crtc) {
2162 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2167 if (!wm_state->cxsr)
2171 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2174 if (num_active_pipes != 1)
2177 if (num_active_pipes > 1)
2178 wm->level = VLV_WM_LEVEL_PM2;
2180 for_each_intel_crtc(&dev_priv->drm, crtc) {
2181 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2182 enum pipe pipe = crtc->pipe;
2184 wm->pipe[pipe] = wm_state->wm[wm->level];
2185 if (crtc->active && wm->cxsr)
2186 wm->sr = wm_state->sr[wm->level];
2188 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2189 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2190 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2191 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2195 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2197 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2198 struct vlv_wm_values new_wm = {};
2200 vlv_merge_wm(dev_priv, &new_wm);
2202 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2205 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2206 chv_set_memory_dvfs(dev_priv, false);
2208 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2209 chv_set_memory_pm5(dev_priv, false);
2211 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2212 _intel_set_memory_cxsr(dev_priv, false);
2214 vlv_write_wm_values(dev_priv, &new_wm);
2216 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2217 _intel_set_memory_cxsr(dev_priv, true);
2219 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2220 chv_set_memory_pm5(dev_priv, true);
2222 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2223 chv_set_memory_dvfs(dev_priv, true);
2228 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2229 struct intel_crtc *crtc)
2231 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2232 const struct intel_crtc_state *crtc_state =
2233 intel_atomic_get_new_crtc_state(state, crtc);
2235 mutex_lock(&dev_priv->wm.wm_mutex);
2236 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2237 vlv_program_watermarks(dev_priv);
2238 mutex_unlock(&dev_priv->wm.wm_mutex);
2241 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2242 struct intel_crtc *crtc)
2244 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2245 const struct intel_crtc_state *crtc_state =
2246 intel_atomic_get_new_crtc_state(state, crtc);
2248 if (!crtc_state->wm.need_postvbl_update)
2251 mutex_lock(&dev_priv->wm.wm_mutex);
2252 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2253 vlv_program_watermarks(dev_priv);
2254 mutex_unlock(&dev_priv->wm.wm_mutex);
2257 static void i965_update_wm(struct drm_i915_private *dev_priv)
2259 struct intel_crtc *crtc;
2264 /* Calc sr entries for one plane configs */
2265 crtc = single_enabled_crtc(dev_priv);
2267 /* self-refresh has much higher latency */
2268 static const int sr_latency_ns = 12000;
2269 const struct drm_display_mode *pipe_mode =
2270 &crtc->config->hw.pipe_mode;
2271 const struct drm_framebuffer *fb =
2272 crtc->base.primary->state->fb;
2273 int pixel_rate = crtc->config->pixel_rate;
2274 int htotal = pipe_mode->crtc_htotal;
2275 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2276 int cpp = fb->format->cpp[0];
2279 entries = intel_wm_method2(pixel_rate, htotal,
2280 width, cpp, sr_latency_ns / 100);
2281 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2282 srwm = I965_FIFO_SIZE - entries;
2286 drm_dbg_kms(&dev_priv->drm,
2287 "self-refresh entries: %d, wm: %d\n",
2290 entries = intel_wm_method2(pixel_rate, htotal,
2291 crtc->base.cursor->state->crtc_w, 4,
2292 sr_latency_ns / 100);
2293 entries = DIV_ROUND_UP(entries,
2294 i965_cursor_wm_info.cacheline_size) +
2295 i965_cursor_wm_info.guard_size;
2297 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2298 if (cursor_sr > i965_cursor_wm_info.max_wm)
2299 cursor_sr = i965_cursor_wm_info.max_wm;
2301 drm_dbg_kms(&dev_priv->drm,
2302 "self-refresh watermark: display plane %d "
2303 "cursor %d\n", srwm, cursor_sr);
2305 cxsr_enabled = true;
2307 cxsr_enabled = false;
2308 /* Turn off self refresh if both pipes are enabled */
2309 intel_set_memory_cxsr(dev_priv, false);
2312 drm_dbg_kms(&dev_priv->drm,
2313 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2316 /* 965 has limitations... */
2317 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2321 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2322 FW_WM(8, PLANEC_OLD));
2323 /* update cursor SR watermark */
2324 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2327 intel_set_memory_cxsr(dev_priv, true);
2332 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2333 enum i9xx_plane_id i9xx_plane)
2335 struct intel_plane *plane;
2337 for_each_intel_plane(&i915->drm, plane) {
2338 if (plane->id == PLANE_PRIMARY &&
2339 plane->i9xx_plane == i9xx_plane)
2340 return intel_crtc_for_pipe(i915, plane->pipe);
2346 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2348 const struct intel_watermark_params *wm_info;
2353 int planea_wm, planeb_wm;
2354 struct intel_crtc *crtc;
2356 if (IS_I945GM(dev_priv))
2357 wm_info = &i945_wm_info;
2358 else if (DISPLAY_VER(dev_priv) != 2)
2359 wm_info = &i915_wm_info;
2361 wm_info = &i830_a_wm_info;
2363 if (DISPLAY_VER(dev_priv) == 2)
2364 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2366 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2367 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2368 if (intel_crtc_active(crtc)) {
2369 const struct drm_framebuffer *fb =
2370 crtc->base.primary->state->fb;
2373 if (DISPLAY_VER(dev_priv) == 2)
2376 cpp = fb->format->cpp[0];
2378 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2379 wm_info, fifo_size, cpp,
2380 pessimal_latency_ns);
2382 planea_wm = fifo_size - wm_info->guard_size;
2383 if (planea_wm > (long)wm_info->max_wm)
2384 planea_wm = wm_info->max_wm;
2387 if (DISPLAY_VER(dev_priv) == 2)
2388 wm_info = &i830_bc_wm_info;
2390 if (DISPLAY_VER(dev_priv) == 2)
2391 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2393 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2394 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2395 if (intel_crtc_active(crtc)) {
2396 const struct drm_framebuffer *fb =
2397 crtc->base.primary->state->fb;
2400 if (DISPLAY_VER(dev_priv) == 2)
2403 cpp = fb->format->cpp[0];
2405 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2406 wm_info, fifo_size, cpp,
2407 pessimal_latency_ns);
2409 planeb_wm = fifo_size - wm_info->guard_size;
2410 if (planeb_wm > (long)wm_info->max_wm)
2411 planeb_wm = wm_info->max_wm;
2414 drm_dbg_kms(&dev_priv->drm,
2415 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2417 crtc = single_enabled_crtc(dev_priv);
2418 if (IS_I915GM(dev_priv) && crtc) {
2419 struct drm_i915_gem_object *obj;
2421 obj = intel_fb_obj(crtc->base.primary->state->fb);
2423 /* self-refresh seems busted with untiled */
2424 if (!i915_gem_object_is_tiled(obj))
2429 * Overlay gets an aggressive default since video jitter is bad.
2433 /* Play safe and disable self-refresh before adjusting watermarks. */
2434 intel_set_memory_cxsr(dev_priv, false);
2436 /* Calc sr entries for one plane configs */
2437 if (HAS_FW_BLC(dev_priv) && crtc) {
2438 /* self-refresh has much higher latency */
2439 static const int sr_latency_ns = 6000;
2440 const struct drm_display_mode *pipe_mode =
2441 &crtc->config->hw.pipe_mode;
2442 const struct drm_framebuffer *fb =
2443 crtc->base.primary->state->fb;
2444 int pixel_rate = crtc->config->pixel_rate;
2445 int htotal = pipe_mode->crtc_htotal;
2446 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2450 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2453 cpp = fb->format->cpp[0];
2455 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2456 sr_latency_ns / 100);
2457 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2458 drm_dbg_kms(&dev_priv->drm,
2459 "self-refresh entries: %d\n", entries);
2460 srwm = wm_info->fifo_size - entries;
2464 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2465 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2466 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2468 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2471 drm_dbg_kms(&dev_priv->drm,
2472 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2473 planea_wm, planeb_wm, cwm, srwm);
2475 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2476 fwater_hi = (cwm & 0x1f);
2478 /* Set request length to 8 cachelines per fetch */
2479 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2480 fwater_hi = fwater_hi | (1 << 8);
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2483 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2486 intel_set_memory_cxsr(dev_priv, true);
2489 static void i845_update_wm(struct drm_i915_private *dev_priv)
2491 struct intel_crtc *crtc;
2495 crtc = single_enabled_crtc(dev_priv);
2499 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2501 i845_get_fifo_size(dev_priv, PLANE_A),
2502 4, pessimal_latency_ns);
2503 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2504 fwater_lo |= (3<<8) | planea_wm;
2506 drm_dbg_kms(&dev_priv->drm,
2507 "Setting FIFO watermarks - A: %d\n", planea_wm);
2509 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2512 /* latency must be in 0.1us units. */
2513 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2515 unsigned int latency)
2519 ret = intel_wm_method1(pixel_rate, cpp, latency);
2520 ret = DIV_ROUND_UP(ret, 64) + 2;
2525 /* latency must be in 0.1us units. */
2526 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2527 unsigned int htotal,
2530 unsigned int latency)
2534 ret = intel_wm_method2(pixel_rate, htotal,
2535 width, cpp, latency);
2536 ret = DIV_ROUND_UP(ret, 64) + 2;
2541 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2544 * Neither of these should be possible since this function shouldn't be
2545 * called if the CRTC is off or the plane is invisible. But let's be
2546 * extra paranoid to avoid a potential divide-by-zero if we screw up
2547 * elsewhere in the driver.
2551 if (WARN_ON(!horiz_pixels))
2554 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2557 struct ilk_wm_maximums {
2565 * For both WM_PIPE and WM_LP.
2566 * mem_value must be in 0.1us units.
2568 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2569 const struct intel_plane_state *plane_state,
2570 u32 mem_value, bool is_lp)
2572 u32 method1, method2;
2578 if (!intel_wm_plane_visible(crtc_state, plane_state))
2581 cpp = plane_state->hw.fb->format->cpp[0];
2583 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2588 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2589 crtc_state->hw.pipe_mode.crtc_htotal,
2590 drm_rect_width(&plane_state->uapi.src) >> 16,
2593 return min(method1, method2);
2597 * For both WM_PIPE and WM_LP.
2598 * mem_value must be in 0.1us units.
2600 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2601 const struct intel_plane_state *plane_state,
2604 u32 method1, method2;
2610 if (!intel_wm_plane_visible(crtc_state, plane_state))
2613 cpp = plane_state->hw.fb->format->cpp[0];
2615 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2616 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2617 crtc_state->hw.pipe_mode.crtc_htotal,
2618 drm_rect_width(&plane_state->uapi.src) >> 16,
2620 return min(method1, method2);
2624 * For both WM_PIPE and WM_LP.
2625 * mem_value must be in 0.1us units.
2627 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2628 const struct intel_plane_state *plane_state,
2636 if (!intel_wm_plane_visible(crtc_state, plane_state))
2639 cpp = plane_state->hw.fb->format->cpp[0];
2641 return ilk_wm_method2(crtc_state->pixel_rate,
2642 crtc_state->hw.pipe_mode.crtc_htotal,
2643 drm_rect_width(&plane_state->uapi.src) >> 16,
2647 /* Only for WM_LP. */
2648 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2649 const struct intel_plane_state *plane_state,
2654 if (!intel_wm_plane_visible(crtc_state, plane_state))
2657 cpp = plane_state->hw.fb->format->cpp[0];
2659 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2664 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2666 if (DISPLAY_VER(dev_priv) >= 8)
2668 else if (DISPLAY_VER(dev_priv) >= 7)
2675 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2676 int level, bool is_sprite)
2678 if (DISPLAY_VER(dev_priv) >= 8)
2679 /* BDW primary/sprite plane watermarks */
2680 return level == 0 ? 255 : 2047;
2681 else if (DISPLAY_VER(dev_priv) >= 7)
2682 /* IVB/HSW primary/sprite plane watermarks */
2683 return level == 0 ? 127 : 1023;
2684 else if (!is_sprite)
2685 /* ILK/SNB primary plane watermarks */
2686 return level == 0 ? 127 : 511;
2688 /* ILK/SNB sprite plane watermarks */
2689 return level == 0 ? 63 : 255;
2693 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2695 if (DISPLAY_VER(dev_priv) >= 7)
2696 return level == 0 ? 63 : 255;
2698 return level == 0 ? 31 : 63;
2701 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2703 if (DISPLAY_VER(dev_priv) >= 8)
2709 /* Calculate the maximum primary/sprite plane watermark */
2710 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2712 const struct intel_wm_config *config,
2713 enum intel_ddb_partitioning ddb_partitioning,
2716 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2718 /* if sprites aren't enabled, sprites get nothing */
2719 if (is_sprite && !config->sprites_enabled)
2722 /* HSW allows LP1+ watermarks even with multiple pipes */
2723 if (level == 0 || config->num_pipes_active > 1) {
2724 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2727 * For some reason the non self refresh
2728 * FIFO size is only half of the self
2729 * refresh FIFO size on ILK/SNB.
2731 if (DISPLAY_VER(dev_priv) <= 6)
2735 if (config->sprites_enabled) {
2736 /* level 0 is always calculated with 1:1 split */
2737 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2746 /* clamp to max that the registers can hold */
2747 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2750 /* Calculate the maximum cursor plane watermark */
2751 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2753 const struct intel_wm_config *config)
2755 /* HSW LP1+ watermarks w/ multiple pipes */
2756 if (level > 0 && config->num_pipes_active > 1)
2759 /* otherwise just report max that registers can hold */
2760 return ilk_cursor_wm_reg_max(dev_priv, level);
2763 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2765 const struct intel_wm_config *config,
2766 enum intel_ddb_partitioning ddb_partitioning,
2767 struct ilk_wm_maximums *max)
2769 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2770 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2771 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2772 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2775 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2777 struct ilk_wm_maximums *max)
2779 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2780 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2781 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2782 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2785 static bool ilk_validate_wm_level(int level,
2786 const struct ilk_wm_maximums *max,
2787 struct intel_wm_level *result)
2791 /* already determined to be invalid? */
2792 if (!result->enable)
2795 result->enable = result->pri_val <= max->pri &&
2796 result->spr_val <= max->spr &&
2797 result->cur_val <= max->cur;
2799 ret = result->enable;
2802 * HACK until we can pre-compute everything,
2803 * and thus fail gracefully if LP0 watermarks
2806 if (level == 0 && !result->enable) {
2807 if (result->pri_val > max->pri)
2808 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2809 level, result->pri_val, max->pri);
2810 if (result->spr_val > max->spr)
2811 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2812 level, result->spr_val, max->spr);
2813 if (result->cur_val > max->cur)
2814 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2815 level, result->cur_val, max->cur);
2817 result->pri_val = min_t(u32, result->pri_val, max->pri);
2818 result->spr_val = min_t(u32, result->spr_val, max->spr);
2819 result->cur_val = min_t(u32, result->cur_val, max->cur);
2820 result->enable = true;
2826 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2827 const struct intel_crtc *crtc,
2829 struct intel_crtc_state *crtc_state,
2830 const struct intel_plane_state *pristate,
2831 const struct intel_plane_state *sprstate,
2832 const struct intel_plane_state *curstate,
2833 struct intel_wm_level *result)
2835 u16 pri_latency = dev_priv->wm.pri_latency[level];
2836 u16 spr_latency = dev_priv->wm.spr_latency[level];
2837 u16 cur_latency = dev_priv->wm.cur_latency[level];
2839 /* WM1+ latency values stored in 0.5us units */
2847 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2848 pri_latency, level);
2849 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2853 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2856 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2858 result->enable = true;
2861 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2864 struct intel_uncore *uncore = &dev_priv->uncore;
2866 if (DISPLAY_VER(dev_priv) >= 9) {
2869 int level, max_level = ilk_wm_max_level(dev_priv);
2870 int mult = IS_DG2(dev_priv) ? 2 : 1;
2872 /* read the first set of memory latencies[0:3] */
2873 val = 0; /* data0 to be programmed to 0 for first set */
2874 ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY,
2878 drm_err(&dev_priv->drm,
2879 "SKL Mailbox read error = %d\n", ret);
2883 wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2884 wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2885 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2886 wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2887 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2888 wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2889 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2891 /* read the second set of memory latencies[4:7] */
2892 val = 1; /* data0 to be programmed to 1 for second set */
2893 ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY,
2896 drm_err(&dev_priv->drm,
2897 "SKL Mailbox read error = %d\n", ret);
2901 wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2902 wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2903 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2904 wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2905 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2906 wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2907 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2910 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2911 * need to be disabled. We make sure to sanitize the values out
2912 * of the punit to satisfy this requirement.
2914 for (level = 1; level <= max_level; level++) {
2915 if (wm[level] == 0) {
2916 for (i = level + 1; i <= max_level; i++)
2919 max_level = level - 1;
2926 * WaWmMemoryReadLatency
2928 * punit doesn't take into account the read latency so we need
2929 * to add proper adjustement to each valid level we retrieve
2930 * from the punit when level 0 response data is 0us.
2933 u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;
2935 for (level = 0; level <= max_level; level++)
2936 wm[level] += adjust;
2940 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2941 * If we could not get dimm info enable this WA to prevent from
2942 * any underrun. If not able to get Dimm info assume 16GB dimm
2943 * to avoid any underrun.
2945 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2947 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2948 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2950 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2952 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2953 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2954 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2955 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2956 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2957 } else if (DISPLAY_VER(dev_priv) >= 6) {
2958 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2960 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2961 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2962 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2963 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2964 } else if (DISPLAY_VER(dev_priv) >= 5) {
2965 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2967 /* ILK primary LP0 latency is 700 ns */
2969 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2970 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2972 MISSING_CASE(INTEL_DEVID(dev_priv));
2976 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2979 /* ILK sprite LP0 latency is 1300 ns */
2980 if (DISPLAY_VER(dev_priv) == 5)
2984 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2987 /* ILK cursor LP0 latency is 1300 ns */
2988 if (DISPLAY_VER(dev_priv) == 5)
2992 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2994 /* how many WM levels are we expecting */
2995 if (HAS_HW_SAGV_WM(dev_priv))
2997 else if (DISPLAY_VER(dev_priv) >= 9)
2999 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3001 else if (DISPLAY_VER(dev_priv) >= 6)
3007 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3011 int level, max_level = ilk_wm_max_level(dev_priv);
3013 for (level = 0; level <= max_level; level++) {
3014 unsigned int latency = wm[level];
3017 drm_dbg_kms(&dev_priv->drm,
3018 "%s WM%d latency not provided\n",
3024 * - latencies are in us on gen9.
3025 * - before then, WM1+ latency values are in 0.5us units
3027 if (DISPLAY_VER(dev_priv) >= 9)
3032 drm_dbg_kms(&dev_priv->drm,
3033 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3034 wm[level], latency / 10, latency % 10);
3038 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3041 int level, max_level = ilk_wm_max_level(dev_priv);
3046 wm[0] = max(wm[0], min);
3047 for (level = 1; level <= max_level; level++)
3048 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3053 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3058 * The BIOS provided WM memory latency values are often
3059 * inadequate for high resolution displays. Adjust them.
3061 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12);
3062 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12);
3063 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3068 drm_dbg_kms(&dev_priv->drm,
3069 "WM latency values increased to avoid potential underruns\n");
3070 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3071 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3072 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3075 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3078 * On some SNB machines (Thinkpad X220 Tablet at least)
3079 * LP3 usage can cause vblank interrupts to be lost.
3080 * The DEIIR bit will go high but it looks like the CPU
3081 * never gets interrupted.
3083 * It's not clear whether other interrupt source could
3084 * be affected or if this is somehow limited to vblank
3085 * interrupts only. To play it safe we disable LP3
3086 * watermarks entirely.
3088 if (dev_priv->wm.pri_latency[3] == 0 &&
3089 dev_priv->wm.spr_latency[3] == 0 &&
3090 dev_priv->wm.cur_latency[3] == 0)
3093 dev_priv->wm.pri_latency[3] = 0;
3094 dev_priv->wm.spr_latency[3] = 0;
3095 dev_priv->wm.cur_latency[3] = 0;
3097 drm_dbg_kms(&dev_priv->drm,
3098 "LP3 watermarks disabled due to potential for lost interrupts\n");
3099 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3100 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3101 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3104 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3106 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3108 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3109 sizeof(dev_priv->wm.pri_latency));
3110 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3111 sizeof(dev_priv->wm.pri_latency));
3113 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3114 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3116 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3117 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3118 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3120 if (DISPLAY_VER(dev_priv) == 6) {
3121 snb_wm_latency_quirk(dev_priv);
3122 snb_wm_lp3_irq_quirk(dev_priv);
3126 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3128 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3129 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3132 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3133 struct intel_pipe_wm *pipe_wm)
3135 /* LP0 watermark maximums depend on this pipe alone */
3136 const struct intel_wm_config config = {
3137 .num_pipes_active = 1,
3138 .sprites_enabled = pipe_wm->sprites_enabled,
3139 .sprites_scaled = pipe_wm->sprites_scaled,
3141 struct ilk_wm_maximums max;
3143 /* LP0 watermarks always use 1/2 DDB partitioning */
3144 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3146 /* At least LP0 must be valid */
3147 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3148 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3155 /* Compute new watermarks for the pipe */
3156 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3157 struct intel_crtc *crtc)
3159 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3160 struct intel_crtc_state *crtc_state =
3161 intel_atomic_get_new_crtc_state(state, crtc);
3162 struct intel_pipe_wm *pipe_wm;
3163 struct intel_plane *plane;
3164 const struct intel_plane_state *plane_state;
3165 const struct intel_plane_state *pristate = NULL;
3166 const struct intel_plane_state *sprstate = NULL;
3167 const struct intel_plane_state *curstate = NULL;
3168 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3169 struct ilk_wm_maximums max;
3171 pipe_wm = &crtc_state->wm.ilk.optimal;
3173 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3174 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3175 pristate = plane_state;
3176 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3177 sprstate = plane_state;
3178 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3179 curstate = plane_state;
3182 pipe_wm->pipe_enabled = crtc_state->hw.active;
3183 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3184 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3186 usable_level = max_level;
3188 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3189 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3192 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3193 if (pipe_wm->sprites_scaled)
3196 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3197 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3198 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3200 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3203 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3205 for (level = 1; level <= usable_level; level++) {
3206 struct intel_wm_level *wm = &pipe_wm->wm[level];
3208 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3209 pristate, sprstate, curstate, wm);
3212 * Disable any watermark level that exceeds the
3213 * register maximums since such watermarks are
3216 if (!ilk_validate_wm_level(level, &max, wm)) {
3217 memset(wm, 0, sizeof(*wm));
3226 * Build a set of 'intermediate' watermark values that satisfy both the old
3227 * state and the new state. These can be programmed to the hardware
3230 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3231 struct intel_crtc *crtc)
3233 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3234 struct intel_crtc_state *new_crtc_state =
3235 intel_atomic_get_new_crtc_state(state, crtc);
3236 const struct intel_crtc_state *old_crtc_state =
3237 intel_atomic_get_old_crtc_state(state, crtc);
3238 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3239 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3240 int level, max_level = ilk_wm_max_level(dev_priv);
3243 * Start with the final, target watermarks, then combine with the
3244 * currently active watermarks to get values that are safe both before
3245 * and after the vblank.
3247 *a = new_crtc_state->wm.ilk.optimal;
3248 if (!new_crtc_state->hw.active ||
3249 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3250 state->skip_intermediate_wm)
3253 a->pipe_enabled |= b->pipe_enabled;
3254 a->sprites_enabled |= b->sprites_enabled;
3255 a->sprites_scaled |= b->sprites_scaled;
3257 for (level = 0; level <= max_level; level++) {
3258 struct intel_wm_level *a_wm = &a->wm[level];
3259 const struct intel_wm_level *b_wm = &b->wm[level];
3261 a_wm->enable &= b_wm->enable;
3262 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3263 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3264 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3265 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3269 * We need to make sure that these merged watermark values are
3270 * actually a valid configuration themselves. If they're not,
3271 * there's no safe way to transition from the old state to
3272 * the new state, so we need to fail the atomic transaction.
3274 if (!ilk_validate_pipe_wm(dev_priv, a))
3278 * If our intermediate WM are identical to the final WM, then we can
3279 * omit the post-vblank programming; only update if it's different.
3281 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3282 new_crtc_state->wm.need_postvbl_update = true;
3288 * Merge the watermarks from all active pipes for a specific level.
3290 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3292 struct intel_wm_level *ret_wm)
3294 const struct intel_crtc *crtc;
3296 ret_wm->enable = true;
3298 for_each_intel_crtc(&dev_priv->drm, crtc) {
3299 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3300 const struct intel_wm_level *wm = &active->wm[level];
3302 if (!active->pipe_enabled)
3306 * The watermark values may have been used in the past,
3307 * so we must maintain them in the registers for some
3308 * time even if the level is now disabled.
3311 ret_wm->enable = false;
3313 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3314 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3315 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3316 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3321 * Merge all low power watermarks for all active pipes.
3323 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3324 const struct intel_wm_config *config,
3325 const struct ilk_wm_maximums *max,
3326 struct intel_pipe_wm *merged)
3328 int level, max_level = ilk_wm_max_level(dev_priv);
3329 int last_enabled_level = max_level;
3331 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3332 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3333 config->num_pipes_active > 1)
3334 last_enabled_level = 0;
3336 /* ILK: FBC WM must be disabled always */
3337 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3339 /* merge each WM1+ level */
3340 for (level = 1; level <= max_level; level++) {
3341 struct intel_wm_level *wm = &merged->wm[level];
3343 ilk_merge_wm_level(dev_priv, level, wm);
3345 if (level > last_enabled_level)
3347 else if (!ilk_validate_wm_level(level, max, wm))
3348 /* make sure all following levels get disabled */
3349 last_enabled_level = level - 1;
3352 * The spec says it is preferred to disable
3353 * FBC WMs instead of disabling a WM level.
3355 if (wm->fbc_val > max->fbc) {
3357 merged->fbc_wm_enabled = false;
3362 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3363 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3364 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3365 for (level = 2; level <= max_level; level++) {
3366 struct intel_wm_level *wm = &merged->wm[level];
3373 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3375 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3376 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3379 /* The value we need to program into the WM_LPx latency field */
3380 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3383 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3386 return dev_priv->wm.pri_latency[level];
3389 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3390 const struct intel_pipe_wm *merged,
3391 enum intel_ddb_partitioning partitioning,
3392 struct ilk_wm_values *results)
3394 struct intel_crtc *crtc;
3397 results->enable_fbc_wm = merged->fbc_wm_enabled;
3398 results->partitioning = partitioning;
3400 /* LP1+ register values */
3401 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3402 const struct intel_wm_level *r;
3404 level = ilk_wm_lp_to_level(wm_lp, merged);
3406 r = &merged->wm[level];
3409 * Maintain the watermark values even if the level is
3410 * disabled. Doing otherwise could cause underruns.
3412 results->wm_lp[wm_lp - 1] =
3413 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3414 WM_LP_PRIMARY(r->pri_val) |
3415 WM_LP_CURSOR(r->cur_val);
3418 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3420 if (DISPLAY_VER(dev_priv) >= 8)
3421 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3423 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3425 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3428 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3429 * level is disabled. Doing otherwise could cause underruns.
3431 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3432 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3433 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3437 /* LP0 register values */
3438 for_each_intel_crtc(&dev_priv->drm, crtc) {
3439 enum pipe pipe = crtc->pipe;
3440 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3441 const struct intel_wm_level *r = &pipe_wm->wm[0];
3443 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3446 results->wm_pipe[pipe] =
3447 WM0_PIPE_PRIMARY(r->pri_val) |
3448 WM0_PIPE_SPRITE(r->spr_val) |
3449 WM0_PIPE_CURSOR(r->cur_val);
3453 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3454 * case both are at the same level. Prefer r1 in case they're the same. */
3455 static struct intel_pipe_wm *
3456 ilk_find_best_result(struct drm_i915_private *dev_priv,
3457 struct intel_pipe_wm *r1,
3458 struct intel_pipe_wm *r2)
3460 int level, max_level = ilk_wm_max_level(dev_priv);
3461 int level1 = 0, level2 = 0;
3463 for (level = 1; level <= max_level; level++) {
3464 if (r1->wm[level].enable)
3466 if (r2->wm[level].enable)
3470 if (level1 == level2) {
3471 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3475 } else if (level1 > level2) {
3482 /* dirty bits used to track which watermarks need changes */
3483 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3484 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3485 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3486 #define WM_DIRTY_FBC (1 << 24)
3487 #define WM_DIRTY_DDB (1 << 25)
3489 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3490 const struct ilk_wm_values *old,
3491 const struct ilk_wm_values *new)
3493 unsigned int dirty = 0;
3497 for_each_pipe(dev_priv, pipe) {
3498 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3499 dirty |= WM_DIRTY_PIPE(pipe);
3500 /* Must disable LP1+ watermarks too */
3501 dirty |= WM_DIRTY_LP_ALL;
3505 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3506 dirty |= WM_DIRTY_FBC;
3507 /* Must disable LP1+ watermarks too */
3508 dirty |= WM_DIRTY_LP_ALL;
3511 if (old->partitioning != new->partitioning) {
3512 dirty |= WM_DIRTY_DDB;
3513 /* Must disable LP1+ watermarks too */
3514 dirty |= WM_DIRTY_LP_ALL;
3517 /* LP1+ watermarks already deemed dirty, no need to continue */
3518 if (dirty & WM_DIRTY_LP_ALL)
3521 /* Find the lowest numbered LP1+ watermark in need of an update... */
3522 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3523 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3524 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3528 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3529 for (; wm_lp <= 3; wm_lp++)
3530 dirty |= WM_DIRTY_LP(wm_lp);
3535 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3538 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3539 bool changed = false;
3541 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3542 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3543 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3546 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3547 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3548 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3551 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3552 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3553 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3558 * Don't touch WM_LP_SPRITE_ENABLE here.
3559 * Doing so could cause underruns.
3566 * The spec says we shouldn't write when we don't need, because every write
3567 * causes WMs to be re-evaluated, expending some power.
3569 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3570 struct ilk_wm_values *results)
3572 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3576 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3580 _ilk_disable_lp_wm(dev_priv, dirty);
3582 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3583 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3584 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3585 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3586 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3587 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3589 if (dirty & WM_DIRTY_DDB) {
3590 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3591 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3592 if (results->partitioning == INTEL_DDB_PART_1_2)
3593 val &= ~WM_MISC_DATA_PARTITION_5_6;
3595 val |= WM_MISC_DATA_PARTITION_5_6;
3596 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3598 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3599 if (results->partitioning == INTEL_DDB_PART_1_2)
3600 val &= ~DISP_DATA_PARTITION_5_6;
3602 val |= DISP_DATA_PARTITION_5_6;
3603 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3607 if (dirty & WM_DIRTY_FBC) {
3608 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3609 if (results->enable_fbc_wm)
3610 val &= ~DISP_FBC_WM_DIS;
3612 val |= DISP_FBC_WM_DIS;
3613 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3616 if (dirty & WM_DIRTY_LP(1) &&
3617 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3618 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3620 if (DISPLAY_VER(dev_priv) >= 7) {
3621 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3622 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3623 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3624 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3627 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3628 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3629 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3630 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3631 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3632 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3634 dev_priv->wm.hw = *results;
3637 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3639 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3642 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3644 u8 enabled_slices = 0;
3645 enum dbuf_slice slice;
3647 for_each_dbuf_slice(dev_priv, slice) {
3648 if (intel_uncore_read(&dev_priv->uncore,
3649 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3650 enabled_slices |= BIT(slice);
3653 return enabled_slices;
3657 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3658 * so assume we'll always need it in order to avoid underruns.
3660 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3662 return DISPLAY_VER(dev_priv) == 9;
3666 intel_has_sagv(struct drm_i915_private *dev_priv)
3668 return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3669 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3673 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3675 if (DISPLAY_VER(dev_priv) >= 12) {
3679 ret = snb_pcode_read(dev_priv,
3680 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3683 dev_priv->sagv_block_time_us = val;
3687 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3688 } else if (DISPLAY_VER(dev_priv) == 11) {
3689 dev_priv->sagv_block_time_us = 10;
3691 } else if (DISPLAY_VER(dev_priv) == 10) {
3692 dev_priv->sagv_block_time_us = 20;
3694 } else if (DISPLAY_VER(dev_priv) == 9) {
3695 dev_priv->sagv_block_time_us = 30;
3698 MISSING_CASE(DISPLAY_VER(dev_priv));
3701 /* Default to an unusable block time */
3702 dev_priv->sagv_block_time_us = -1;
3706 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3707 * depending on power and performance requirements. The display engine access
3708 * to system memory is blocked during the adjustment time. Because of the
3709 * blocking time, having this enabled can cause full system hangs and/or pipe
3710 * underruns if we don't meet all of the following requirements:
3712 * - <= 1 pipe enabled
3713 * - All planes can enable watermarks for latencies >= SAGV engine block time
3714 * - We're not using an interlaced display configuration
3717 intel_enable_sagv(struct drm_i915_private *dev_priv)
3721 if (!intel_has_sagv(dev_priv))
3724 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3727 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3728 ret = snb_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3731 /* We don't need to wait for SAGV when enabling */
3734 * Some skl systems, pre-release machines in particular,
3735 * don't actually have SAGV.
3737 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3738 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3739 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3741 } else if (ret < 0) {
3742 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3746 dev_priv->sagv_status = I915_SAGV_ENABLED;
3751 intel_disable_sagv(struct drm_i915_private *dev_priv)
3755 if (!intel_has_sagv(dev_priv))
3758 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3761 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3762 /* bspec says to keep retrying for at least 1 ms */
3763 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3765 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3768 * Some skl systems, pre-release machines in particular,
3769 * don't actually have SAGV.
3771 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3772 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3773 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3775 } else if (ret < 0) {
3776 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3780 dev_priv->sagv_status = I915_SAGV_DISABLED;
3784 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
3786 struct drm_i915_private *i915 = to_i915(state->base.dev);
3787 const struct intel_bw_state *new_bw_state =
3788 intel_atomic_get_new_bw_state(state);
3793 if (!intel_can_enable_sagv(i915, new_bw_state))
3794 intel_disable_sagv(i915);
3797 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
3799 struct drm_i915_private *i915 = to_i915(state->base.dev);
3800 const struct intel_bw_state *new_bw_state =
3801 intel_atomic_get_new_bw_state(state);
3806 if (intel_can_enable_sagv(i915, new_bw_state))
3807 intel_enable_sagv(i915);
3810 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
3812 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3813 const struct intel_bw_state *old_bw_state =
3814 intel_atomic_get_old_bw_state(state);
3815 const struct intel_bw_state *new_bw_state =
3816 intel_atomic_get_new_bw_state(state);
3817 u16 old_mask, new_mask;
3822 old_mask = old_bw_state->qgv_points_mask;
3823 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3825 if (old_mask == new_mask)
3828 WARN_ON(!new_bw_state->base.changed);
3830 drm_dbg_kms(&dev_priv->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
3831 old_mask, new_mask);
3834 * Restrict required qgv points before updating the configuration.
3835 * According to BSpec we can't mask and unmask qgv points at the same
3836 * time. Also masking should be done before updating the configuration
3837 * and unmasking afterwards.
3839 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3842 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
3844 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3845 const struct intel_bw_state *old_bw_state =
3846 intel_atomic_get_old_bw_state(state);
3847 const struct intel_bw_state *new_bw_state =
3848 intel_atomic_get_new_bw_state(state);
3849 u16 old_mask, new_mask;
3854 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3855 new_mask = new_bw_state->qgv_points_mask;
3857 if (old_mask == new_mask)
3860 WARN_ON(!new_bw_state->base.changed);
3862 drm_dbg_kms(&dev_priv->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
3863 old_mask, new_mask);
3866 * Allow required qgv points after updating the configuration.
3867 * According to BSpec we can't mask and unmask qgv points at the same
3868 * time. Also masking should be done before updating the configuration
3869 * and unmasking afterwards.
3871 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3874 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3876 struct drm_i915_private *i915 = to_i915(state->base.dev);
3879 * Just return if we can't control SAGV or don't have it.
3880 * This is different from situation when we have SAGV but just can't
3881 * afford it due to DBuf limitation - in case if SAGV is completely
3882 * disabled in a BIOS, we are not even allowed to send a PCode request,
3883 * as it will throw an error. So have to check it here.
3885 if (!intel_has_sagv(i915))
3888 if (DISPLAY_VER(i915) >= 11)
3889 icl_sagv_pre_plane_update(state);
3891 skl_sagv_pre_plane_update(state);
3894 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3896 struct drm_i915_private *i915 = to_i915(state->base.dev);
3899 * Just return if we can't control SAGV or don't have it.
3900 * This is different from situation when we have SAGV but just can't
3901 * afford it due to DBuf limitation - in case if SAGV is completely
3902 * disabled in a BIOS, we are not even allowed to send a PCode request,
3903 * as it will throw an error. So have to check it here.
3905 if (!intel_has_sagv(i915))
3908 if (DISPLAY_VER(i915) >= 11)
3909 icl_sagv_post_plane_update(state);
3911 skl_sagv_post_plane_update(state);
3914 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3916 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3917 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3918 enum plane_id plane_id;
3919 int max_level = INT_MAX;
3921 if (!intel_has_sagv(dev_priv))
3924 if (!crtc_state->hw.active)
3927 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3930 for_each_plane_id_on_crtc(crtc, plane_id) {
3931 const struct skl_plane_wm *wm =
3932 &crtc_state->wm.skl.optimal.planes[plane_id];
3935 /* Skip this plane if it's not enabled */
3936 if (!wm->wm[0].enable)
3939 /* Find the highest enabled wm level for this plane */
3940 for (level = ilk_wm_max_level(dev_priv);
3941 !wm->wm[level].enable; --level)
3944 /* Highest common enabled wm level for all planes */
3945 max_level = min(level, max_level);
3948 /* No enabled planes? */
3949 if (max_level == INT_MAX)
3952 for_each_plane_id_on_crtc(crtc, plane_id) {
3953 const struct skl_plane_wm *wm =
3954 &crtc_state->wm.skl.optimal.planes[plane_id];
3957 * All enabled planes must have enabled a common wm level that
3958 * can tolerate memory latencies higher than sagv_block_time_us
3960 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3967 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3969 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3970 enum plane_id plane_id;
3972 if (!crtc_state->hw.active)
3975 for_each_plane_id_on_crtc(crtc, plane_id) {
3976 const struct skl_plane_wm *wm =
3977 &crtc_state->wm.skl.optimal.planes[plane_id];
3979 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3986 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3988 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3989 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3991 if (DISPLAY_VER(dev_priv) >= 12)
3992 return tgl_crtc_can_enable_sagv(crtc_state);
3994 return skl_crtc_can_enable_sagv(crtc_state);
3997 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3998 const struct intel_bw_state *bw_state)
4000 if (DISPLAY_VER(dev_priv) < 11 &&
4001 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
4004 return bw_state->pipe_sagv_reject == 0;
4007 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
4009 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4011 struct intel_crtc *crtc;
4012 struct intel_crtc_state *new_crtc_state;
4013 struct intel_bw_state *new_bw_state = NULL;
4014 const struct intel_bw_state *old_bw_state = NULL;
4017 for_each_new_intel_crtc_in_state(state, crtc,
4018 new_crtc_state, i) {
4019 new_bw_state = intel_atomic_get_bw_state(state);
4020 if (IS_ERR(new_bw_state))
4021 return PTR_ERR(new_bw_state);
4023 old_bw_state = intel_atomic_get_old_bw_state(state);
4025 if (intel_crtc_can_enable_sagv(new_crtc_state))
4026 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4028 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4034 new_bw_state->active_pipes =
4035 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4037 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4038 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4043 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4044 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4045 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4048 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4049 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4054 for_each_new_intel_crtc_in_state(state, crtc,
4055 new_crtc_state, i) {
4056 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4059 * We store use_sagv_wm in the crtc state rather than relying on
4060 * that bw state since we have no convenient way to get at the
4061 * latter from the plane commit hooks (especially in the legacy
4064 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4065 DISPLAY_VER(dev_priv) >= 12 &&
4066 intel_can_enable_sagv(dev_priv, new_bw_state);
4072 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
4075 entry->start = start;
4081 static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4083 return INTEL_INFO(dev_priv)->dbuf.size /
4084 hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4088 skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4089 struct skl_ddb_entry *ddb)
4091 int slice_size = intel_dbuf_slice_size(dev_priv);
4099 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4100 ddb->end = fls(slice_mask) * slice_size;
4102 WARN_ON(ddb->start >= ddb->end);
4103 WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4106 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4108 struct skl_ddb_entry ddb;
4110 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4111 slice_mask = BIT(DBUF_S1);
4112 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4113 slice_mask = BIT(DBUF_S3);
4115 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4120 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4121 const struct skl_ddb_entry *entry)
4123 int slice_size = intel_dbuf_slice_size(dev_priv);
4124 enum dbuf_slice start_slice, end_slice;
4127 if (!skl_ddb_entry_size(entry))
4130 start_slice = entry->start / slice_size;
4131 end_slice = (entry->end - 1) / slice_size;
4134 * Per plane DDB entry can in a really worst case be on multiple slices
4135 * but single entry is anyway contigious.
4137 while (start_slice <= end_slice) {
4138 slice_mask |= BIT(start_slice);
4145 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4147 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4148 int hdisplay, vdisplay;
4150 if (!crtc_state->hw.active)
4154 * Watermark/ddb requirement highly depends upon width of the
4155 * framebuffer, So instead of allocating DDB equally among pipes
4156 * distribute DDB based on resolution/width of the display.
4158 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4163 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4165 unsigned int *weight_start,
4166 unsigned int *weight_end,
4167 unsigned int *weight_total)
4169 struct drm_i915_private *dev_priv =
4170 to_i915(dbuf_state->base.state->base.dev);
4177 for_each_pipe(dev_priv, pipe) {
4178 int weight = dbuf_state->weight[pipe];
4181 * Do not account pipes using other slice sets
4182 * luckily as of current BSpec slice sets do not partially
4183 * intersect(pipes share either same one slice or same slice set
4184 * i.e no partial intersection), so it is enough to check for
4187 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4190 *weight_total += weight;
4191 if (pipe < for_pipe) {
4192 *weight_start += weight;
4193 *weight_end += weight;
4194 } else if (pipe == for_pipe) {
4195 *weight_end += weight;
4201 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4203 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4204 unsigned int weight_total, weight_start, weight_end;
4205 const struct intel_dbuf_state *old_dbuf_state =
4206 intel_atomic_get_old_dbuf_state(state);
4207 struct intel_dbuf_state *new_dbuf_state =
4208 intel_atomic_get_new_dbuf_state(state);
4209 struct intel_crtc_state *crtc_state;
4210 struct skl_ddb_entry ddb_slices;
4211 enum pipe pipe = crtc->pipe;
4212 unsigned int mbus_offset = 0;
4214 u32 dbuf_slice_mask;
4218 if (new_dbuf_state->weight[pipe] == 0) {
4219 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
4223 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4225 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4226 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4227 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4229 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4230 &weight_start, &weight_end, &weight_total);
4232 start = ddb_range_size * weight_start / weight_total;
4233 end = ddb_range_size * weight_end / weight_total;
4235 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
4236 ddb_slices.start - mbus_offset + start,
4237 ddb_slices.start - mbus_offset + end);
4240 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4241 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4242 &new_dbuf_state->ddb[pipe]))
4245 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4249 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4250 if (IS_ERR(crtc_state))
4251 return PTR_ERR(crtc_state);
4254 * Used for checking overlaps, so we need absolute
4255 * offsets instead of MBUS relative offsets.
4257 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4258 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4260 drm_dbg_kms(&dev_priv->drm,
4261 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4262 crtc->base.base.id, crtc->base.name,
4263 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4264 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4265 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4266 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4271 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4272 int width, const struct drm_format_info *format,
4273 u64 modifier, unsigned int rotation,
4274 u32 plane_pixel_rate, struct skl_wm_params *wp,
4277 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4278 struct intel_plane *plane,
4280 unsigned int latency,
4281 const struct skl_wm_params *wp,
4282 const struct skl_wm_level *result_prev,
4283 struct skl_wm_level *result /* out */);
4286 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4289 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
4290 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4291 int level, max_level = ilk_wm_max_level(dev_priv);
4292 struct skl_wm_level wm = {};
4293 int ret, min_ddb_alloc = 0;
4294 struct skl_wm_params wp;
4296 ret = skl_compute_wm_params(crtc_state, 256,
4297 drm_format_info(DRM_FORMAT_ARGB8888),
4298 DRM_FORMAT_MOD_LINEAR,
4300 crtc_state->pixel_rate, &wp, 0);
4301 drm_WARN_ON(&dev_priv->drm, ret);
4303 for (level = 0; level <= max_level; level++) {
4304 unsigned int latency = dev_priv->wm.skl_latency[level];
4306 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
4307 if (wm.min_ddb_alloc == U16_MAX)
4310 min_ddb_alloc = wm.min_ddb_alloc;
4313 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4316 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
4318 skl_ddb_entry_init(entry,
4319 REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
4320 REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
4326 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4327 const enum pipe pipe,
4328 const enum plane_id plane_id,
4329 struct skl_ddb_entry *ddb_y,
4330 struct skl_ddb_entry *ddb_uv)
4335 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4336 if (plane_id == PLANE_CURSOR) {
4337 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4338 skl_ddb_entry_init_from_hw(ddb_y, val);
4342 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4344 /* No DDB allocated for disabled planes */
4345 if (val & PLANE_CTL_ENABLE)
4346 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK_SKL,
4347 val & PLANE_CTL_ORDER_RGBX,
4348 val & PLANE_CTL_ALPHA_MASK);
4350 if (DISPLAY_VER(dev_priv) >= 11) {
4351 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4352 skl_ddb_entry_init_from_hw(ddb_y, val);
4354 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4355 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4358 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4361 skl_ddb_entry_init_from_hw(ddb_y, val);
4362 skl_ddb_entry_init_from_hw(ddb_uv, val2);
4366 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4367 struct skl_ddb_entry *ddb_y,
4368 struct skl_ddb_entry *ddb_uv)
4370 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4371 enum intel_display_power_domain power_domain;
4372 enum pipe pipe = crtc->pipe;
4373 intel_wakeref_t wakeref;
4374 enum plane_id plane_id;
4376 power_domain = POWER_DOMAIN_PIPE(pipe);
4377 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4381 for_each_plane_id_on_crtc(crtc, plane_id)
4382 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4387 intel_display_power_put(dev_priv, power_domain, wakeref);
4390 struct dbuf_slice_conf_entry {
4392 u8 dbuf_mask[I915_MAX_PIPES];
4397 * Table taken from Bspec 12716
4398 * Pipes do have some preferred DBuf slice affinity,
4399 * plus there are some hardcoded requirements on how
4400 * those should be distributed for multipipe scenarios.
4401 * For more DBuf slices algorithm can get even more messy
4402 * and less readable, so decided to use a table almost
4403 * as is from BSpec itself - that way it is at least easier
4404 * to compare, change and check.
4406 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4407 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4410 .active_pipes = BIT(PIPE_A),
4412 [PIPE_A] = BIT(DBUF_S1),
4416 .active_pipes = BIT(PIPE_B),
4418 [PIPE_B] = BIT(DBUF_S1),
4422 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4424 [PIPE_A] = BIT(DBUF_S1),
4425 [PIPE_B] = BIT(DBUF_S2),
4429 .active_pipes = BIT(PIPE_C),
4431 [PIPE_C] = BIT(DBUF_S2),
4435 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4437 [PIPE_A] = BIT(DBUF_S1),
4438 [PIPE_C] = BIT(DBUF_S2),
4442 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4444 [PIPE_B] = BIT(DBUF_S1),
4445 [PIPE_C] = BIT(DBUF_S2),
4449 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4451 [PIPE_A] = BIT(DBUF_S1),
4452 [PIPE_B] = BIT(DBUF_S1),
4453 [PIPE_C] = BIT(DBUF_S2),
4460 * Table taken from Bspec 49255
4461 * Pipes do have some preferred DBuf slice affinity,
4462 * plus there are some hardcoded requirements on how
4463 * those should be distributed for multipipe scenarios.
4464 * For more DBuf slices algorithm can get even more messy
4465 * and less readable, so decided to use a table almost
4466 * as is from BSpec itself - that way it is at least easier
4467 * to compare, change and check.
4469 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4470 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4473 .active_pipes = BIT(PIPE_A),
4475 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4479 .active_pipes = BIT(PIPE_B),
4481 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4485 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4487 [PIPE_A] = BIT(DBUF_S2),
4488 [PIPE_B] = BIT(DBUF_S1),
4492 .active_pipes = BIT(PIPE_C),
4494 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4498 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4500 [PIPE_A] = BIT(DBUF_S1),
4501 [PIPE_C] = BIT(DBUF_S2),
4505 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4507 [PIPE_B] = BIT(DBUF_S1),
4508 [PIPE_C] = BIT(DBUF_S2),
4512 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4514 [PIPE_A] = BIT(DBUF_S1),
4515 [PIPE_B] = BIT(DBUF_S1),
4516 [PIPE_C] = BIT(DBUF_S2),
4520 .active_pipes = BIT(PIPE_D),
4522 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4526 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4528 [PIPE_A] = BIT(DBUF_S1),
4529 [PIPE_D] = BIT(DBUF_S2),
4533 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4535 [PIPE_B] = BIT(DBUF_S1),
4536 [PIPE_D] = BIT(DBUF_S2),
4540 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4542 [PIPE_A] = BIT(DBUF_S1),
4543 [PIPE_B] = BIT(DBUF_S1),
4544 [PIPE_D] = BIT(DBUF_S2),
4548 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4550 [PIPE_C] = BIT(DBUF_S1),
4551 [PIPE_D] = BIT(DBUF_S2),
4555 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4557 [PIPE_A] = BIT(DBUF_S1),
4558 [PIPE_C] = BIT(DBUF_S2),
4559 [PIPE_D] = BIT(DBUF_S2),
4563 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4565 [PIPE_B] = BIT(DBUF_S1),
4566 [PIPE_C] = BIT(DBUF_S2),
4567 [PIPE_D] = BIT(DBUF_S2),
4571 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4573 [PIPE_A] = BIT(DBUF_S1),
4574 [PIPE_B] = BIT(DBUF_S1),
4575 [PIPE_C] = BIT(DBUF_S2),
4576 [PIPE_D] = BIT(DBUF_S2),
4582 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
4584 .active_pipes = BIT(PIPE_A),
4586 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4590 .active_pipes = BIT(PIPE_B),
4592 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4596 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4598 [PIPE_A] = BIT(DBUF_S1),
4599 [PIPE_B] = BIT(DBUF_S2),
4603 .active_pipes = BIT(PIPE_C),
4605 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4609 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4611 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4612 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4616 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4618 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4619 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4623 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4625 [PIPE_A] = BIT(DBUF_S1),
4626 [PIPE_B] = BIT(DBUF_S2),
4627 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4631 .active_pipes = BIT(PIPE_D),
4633 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4637 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4639 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4640 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4644 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4646 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4647 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4651 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4653 [PIPE_A] = BIT(DBUF_S1),
4654 [PIPE_B] = BIT(DBUF_S2),
4655 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4659 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4661 [PIPE_C] = BIT(DBUF_S3),
4662 [PIPE_D] = BIT(DBUF_S4),
4666 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4668 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4669 [PIPE_C] = BIT(DBUF_S3),
4670 [PIPE_D] = BIT(DBUF_S4),
4674 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4676 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4677 [PIPE_C] = BIT(DBUF_S3),
4678 [PIPE_D] = BIT(DBUF_S4),
4682 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4684 [PIPE_A] = BIT(DBUF_S1),
4685 [PIPE_B] = BIT(DBUF_S2),
4686 [PIPE_C] = BIT(DBUF_S3),
4687 [PIPE_D] = BIT(DBUF_S4),
4693 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4695 * Keep the join_mbus cases first so check_mbus_joined()
4696 * will prefer them over the !join_mbus cases.
4699 .active_pipes = BIT(PIPE_A),
4701 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4706 .active_pipes = BIT(PIPE_B),
4708 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4713 .active_pipes = BIT(PIPE_A),
4715 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4720 .active_pipes = BIT(PIPE_B),
4722 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4727 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4729 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4730 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4734 .active_pipes = BIT(PIPE_C),
4736 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4740 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4742 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4743 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4747 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4749 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4750 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4754 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4756 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4757 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4758 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4762 .active_pipes = BIT(PIPE_D),
4764 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4768 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4770 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4771 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4775 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4777 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4778 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4782 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4784 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4785 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4786 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4790 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4792 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4793 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4797 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4799 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4800 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4801 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4805 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4807 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4808 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4809 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4813 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4815 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4816 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4817 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4818 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4825 static bool check_mbus_joined(u8 active_pipes,
4826 const struct dbuf_slice_conf_entry *dbuf_slices)
4830 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4831 if (dbuf_slices[i].active_pipes == active_pipes)
4832 return dbuf_slices[i].join_mbus;
4837 static bool adlp_check_mbus_joined(u8 active_pipes)
4839 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4842 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
4843 const struct dbuf_slice_conf_entry *dbuf_slices)
4847 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4848 if (dbuf_slices[i].active_pipes == active_pipes &&
4849 dbuf_slices[i].join_mbus == join_mbus)
4850 return dbuf_slices[i].dbuf_mask[pipe];
4856 * This function finds an entry with same enabled pipe configuration and
4857 * returns correspondent DBuf slice mask as stated in BSpec for particular
4860 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4863 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4864 * required calculating "pipe ratio" in order to determine
4865 * if one or two slices can be used for single pipe configurations
4866 * as additional constraint to the existing table.
4867 * However based on recent info, it should be not "pipe ratio"
4868 * but rather ratio between pixel_rate and cdclk with additional
4869 * constants, so for now we are using only table until this is
4870 * clarified. Also this is the reason why crtc_state param is
4871 * still here - we will need it once those additional constraints
4874 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4878 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4880 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4884 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4886 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4887 adlp_allowed_dbufs);
4890 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4892 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4896 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
4898 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4899 enum pipe pipe = crtc->pipe;
4901 if (IS_DG2(dev_priv))
4902 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4903 else if (IS_ALDERLAKE_P(dev_priv))
4904 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4905 else if (DISPLAY_VER(dev_priv) == 12)
4906 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4907 else if (DISPLAY_VER(dev_priv) == 11)
4908 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4910 * For anything else just return one slice yet.
4911 * Should be extended for other platforms.
4913 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4917 use_min_ddb(const struct intel_crtc_state *crtc_state,
4918 struct intel_plane *plane)
4920 struct drm_i915_private *i915 = to_i915(plane->base.dev);
4922 return DISPLAY_VER(i915) >= 13 &&
4923 crtc_state->uapi.async_flip &&
4928 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
4929 struct intel_plane *plane)
4931 struct drm_i915_private *i915 = to_i915(plane->base.dev);
4933 return DISPLAY_VER(i915) >= 13 &&
4934 crtc_state->uapi.async_flip &&
4939 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4940 const struct intel_plane_state *plane_state,
4943 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4944 const struct drm_framebuffer *fb = plane_state->hw.fb;
4947 if (!plane_state->uapi.visible)
4950 if (plane->id == PLANE_CURSOR)
4954 * We calculate extra ddb based on ratio plane rate/total data rate
4955 * in case, in some cases we should not allocate extra ddb for the plane,
4956 * so do not count its data rate, if this is the case.
4958 if (use_min_ddb(crtc_state, plane))
4961 if (color_plane == 1 &&
4962 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4966 * Src coordinates are already rotated by 270 degrees for
4967 * the 90/270 degree plane rotation cases (to match the
4968 * GTT mapping), hence no need to account for rotation here.
4970 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4971 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4973 /* UV plane does 1/2 pixel sub-sampling */
4974 if (color_plane == 1) {
4979 return width * height * fb->format->cpp[color_plane];
4983 skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4984 struct intel_crtc *crtc)
4986 struct intel_crtc_state *crtc_state =
4987 intel_atomic_get_new_crtc_state(state, crtc);
4988 const struct intel_plane_state *plane_state;
4989 struct intel_plane *plane;
4990 u64 total_data_rate = 0;
4991 enum plane_id plane_id;
4994 /* Calculate and cache data rate for each plane */
4995 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4996 if (plane->pipe != crtc->pipe)
4999 plane_id = plane->id;
5002 crtc_state->plane_data_rate[plane_id] =
5003 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5006 crtc_state->uv_plane_data_rate[plane_id] =
5007 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
5010 for_each_plane_id_on_crtc(crtc, plane_id) {
5011 total_data_rate += crtc_state->plane_data_rate[plane_id];
5012 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
5015 return total_data_rate;
5019 icl_get_total_relative_data_rate(struct intel_atomic_state *state,
5020 struct intel_crtc *crtc)
5022 struct intel_crtc_state *crtc_state =
5023 intel_atomic_get_new_crtc_state(state, crtc);
5024 const struct intel_plane_state *plane_state;
5025 struct intel_plane *plane;
5026 u64 total_data_rate = 0;
5027 enum plane_id plane_id;
5030 /* Calculate and cache data rate for each plane */
5031 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5032 if (plane->pipe != crtc->pipe)
5035 plane_id = plane->id;
5037 if (!plane_state->planar_linked_plane) {
5038 crtc_state->plane_data_rate[plane_id] =
5039 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5041 enum plane_id y_plane_id;
5044 * The slave plane might not iterate in
5045 * intel_atomic_crtc_state_for_each_plane_state(),
5046 * and needs the master plane state which may be
5047 * NULL if we try get_new_plane_state(), so we
5048 * always calculate from the master.
5050 if (plane_state->planar_slave)
5053 /* Y plane rate is calculated on the slave */
5054 y_plane_id = plane_state->planar_linked_plane->id;
5055 crtc_state->plane_data_rate[y_plane_id] =
5056 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5058 crtc_state->plane_data_rate[plane_id] =
5059 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
5063 for_each_plane_id_on_crtc(crtc, plane_id)
5064 total_data_rate += crtc_state->plane_data_rate[plane_id];
5066 return total_data_rate;
5069 const struct skl_wm_level *
5070 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
5071 enum plane_id plane_id,
5074 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5076 if (level == 0 && pipe_wm->use_sagv_wm)
5077 return &wm->sagv.wm0;
5079 return &wm->wm[level];
5082 const struct skl_wm_level *
5083 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
5084 enum plane_id plane_id)
5086 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5088 if (pipe_wm->use_sagv_wm)
5089 return &wm->sagv.trans_wm;
5091 return &wm->trans_wm;
5095 * We only disable the watermarks for each plane if
5096 * they exceed the ddb allocation of said plane. This
5097 * is done so that we don't end up touching cursor
5098 * watermarks needlessly when some other plane reduces
5099 * our max possible watermark level.
5101 * Bspec has this to say about the PLANE_WM enable bit:
5102 * "All the watermarks at this level for all enabled
5103 * planes must be enabled before the level will be used."
5104 * So this is actually safe to do.
5107 skl_check_wm_level(struct skl_wm_level *wm, u64 total)
5109 if (wm->min_ddb_alloc > total)
5110 memset(wm, 0, sizeof(*wm));
5114 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
5115 u64 total, u64 uv_total)
5117 if (wm->min_ddb_alloc > total ||
5118 uv_wm->min_ddb_alloc > uv_total) {
5119 memset(wm, 0, sizeof(*wm));
5120 memset(uv_wm, 0, sizeof(*uv_wm));
5124 static bool icl_need_wm1_wa(struct drm_i915_private *i915,
5125 enum plane_id plane_id)
5128 * Wa_1408961008:icl, ehl
5129 * Wa_14012656716:tgl, adl
5130 * Underruns with WM1+ disabled
5132 return DISPLAY_VER(i915) == 11 ||
5133 (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
5136 struct skl_plane_ddb_iter {
5138 u16 total[I915_MAX_PLANES];
5139 u16 uv_total[I915_MAX_PLANES];
5144 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
5145 const struct skl_wm_level *wm,
5151 extra = min_t(u16, iter->size,
5152 DIV64_U64_ROUND_UP(iter->size * data_rate,
5154 iter->size -= extra;
5155 iter->data_rate -= data_rate;
5158 return wm->min_ddb_alloc + extra;
5162 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
5163 struct intel_crtc *crtc)
5165 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5166 struct intel_crtc_state *crtc_state =
5167 intel_atomic_get_new_crtc_state(state, crtc);
5168 const struct intel_dbuf_state *dbuf_state =
5169 intel_atomic_get_new_dbuf_state(state);
5170 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
5171 int num_active = hweight8(dbuf_state->active_pipes);
5172 struct skl_plane_ddb_iter iter = {};
5173 enum plane_id plane_id;
5177 /* Clear the partitioning for disabled planes. */
5178 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
5179 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
5181 if (!crtc_state->hw.active)
5184 if (DISPLAY_VER(dev_priv) >= 11)
5185 iter.data_rate = icl_get_total_relative_data_rate(state, crtc);
5187 iter.data_rate = skl_get_total_relative_data_rate(state, crtc);
5189 iter.size = skl_ddb_entry_size(alloc);
5193 /* Allocate fixed number of blocks for cursor. */
5194 iter.total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
5195 iter.size -= iter.total[PLANE_CURSOR];
5196 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR],
5197 alloc->end - iter.total[PLANE_CURSOR], alloc->end);
5200 * Find the highest watermark level for which we can satisfy the block
5201 * requirement of active planes.
5203 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5205 for_each_plane_id_on_crtc(crtc, plane_id) {
5206 const struct skl_plane_wm *wm =
5207 &crtc_state->wm.skl.optimal.planes[plane_id];
5209 if (plane_id == PLANE_CURSOR) {
5210 if (wm->wm[level].min_ddb_alloc > iter.total[PLANE_CURSOR]) {
5211 drm_WARN_ON(&dev_priv->drm,
5212 wm->wm[level].min_ddb_alloc != U16_MAX);
5219 blocks += wm->wm[level].min_ddb_alloc;
5220 blocks += wm->uv_wm[level].min_ddb_alloc;
5223 if (blocks <= iter.size) {
5224 iter.size -= blocks;
5230 drm_dbg_kms(&dev_priv->drm,
5231 "Requested display configuration exceeds system DDB limitations");
5232 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5237 /* avoid the WARN later when we don't allocate any extra DDB */
5238 if (iter.data_rate == 0)
5242 * Grant each plane the blocks it requires at the highest achievable
5243 * watermark level, plus an extra share of the leftover blocks
5244 * proportional to its relative data rate.
5246 for_each_plane_id_on_crtc(crtc, plane_id) {
5247 const struct skl_plane_wm *wm =
5248 &crtc_state->wm.skl.optimal.planes[plane_id];
5250 if (plane_id == PLANE_CURSOR)
5253 iter.total[plane_id] =
5254 skl_allocate_plane_ddb(&iter, &wm->wm[level],
5255 crtc_state->plane_data_rate[plane_id]);
5257 iter.uv_total[plane_id] =
5258 skl_allocate_plane_ddb(&iter, &wm->uv_wm[level],
5259 crtc_state->uv_plane_data_rate[plane_id]);
5261 drm_WARN_ON(&dev_priv->drm, iter.size != 0 || iter.data_rate != 0);
5263 /* Set the actual DDB start/end points for each plane */
5264 iter.start = alloc->start;
5265 for_each_plane_id_on_crtc(crtc, plane_id) {
5266 struct skl_ddb_entry *plane_alloc =
5267 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5268 struct skl_ddb_entry *uv_plane_alloc =
5269 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5271 if (plane_id == PLANE_CURSOR)
5274 /* Gen11+ uses a separate plane for UV watermarks */
5275 drm_WARN_ON(&dev_priv->drm,
5276 DISPLAY_VER(dev_priv) >= 11 && iter.uv_total[plane_id]);
5278 /* Leave disabled planes at (0,0) */
5279 if (iter.total[plane_id])
5280 iter.start = skl_ddb_entry_init(plane_alloc, iter.start,
5281 iter.start + iter.total[plane_id]);
5283 if (iter.uv_total[plane_id])
5284 iter.start = skl_ddb_entry_init(uv_plane_alloc, iter.start,
5285 iter.start + iter.uv_total[plane_id]);
5289 * When we calculated watermark values we didn't know how high
5290 * of a level we'd actually be able to hit, so we just marked
5291 * all levels as "enabled." Go back now and disable the ones
5292 * that aren't actually possible.
5294 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5295 for_each_plane_id_on_crtc(crtc, plane_id) {
5296 struct skl_plane_wm *wm =
5297 &crtc_state->wm.skl.optimal.planes[plane_id];
5299 skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
5300 iter.total[plane_id],
5301 iter.uv_total[plane_id]);
5303 if (icl_need_wm1_wa(dev_priv, plane_id) &&
5304 level == 1 && wm->wm[0].enable) {
5305 wm->wm[level].blocks = wm->wm[0].blocks;
5306 wm->wm[level].lines = wm->wm[0].lines;
5307 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5313 * Go back and disable the transition and SAGV watermarks
5314 * if it turns out we don't have enough DDB blocks for them.
5316 for_each_plane_id_on_crtc(crtc, plane_id) {
5317 struct skl_plane_wm *wm =
5318 &crtc_state->wm.skl.optimal.planes[plane_id];
5320 skl_check_wm_level(&wm->trans_wm, iter.total[plane_id]);
5321 skl_check_wm_level(&wm->sagv.wm0, iter.total[plane_id]);
5322 skl_check_wm_level(&wm->sagv.trans_wm, iter.total[plane_id]);
5329 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5330 * for the read latency) and cpp should always be <= 8, so that
5331 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5332 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5334 static uint_fixed_16_16_t
5335 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5336 u8 cpp, u32 latency, u32 dbuf_block_size)
5338 u32 wm_intermediate_val;
5339 uint_fixed_16_16_t ret;
5342 return FP_16_16_MAX;
5344 wm_intermediate_val = latency * pixel_rate * cpp;
5345 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5347 if (DISPLAY_VER(dev_priv) >= 10)
5348 ret = add_fixed16_u32(ret, 1);
5353 static uint_fixed_16_16_t
5354 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5355 uint_fixed_16_16_t plane_blocks_per_line)
5357 u32 wm_intermediate_val;
5358 uint_fixed_16_16_t ret;
5361 return FP_16_16_MAX;
5363 wm_intermediate_val = latency * pixel_rate;
5364 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5365 pipe_htotal * 1000);
5366 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5370 static uint_fixed_16_16_t
5371 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5373 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5376 uint_fixed_16_16_t linetime_us;
5378 if (!crtc_state->hw.active)
5379 return u32_to_fixed16(0);
5381 pixel_rate = crtc_state->pixel_rate;
5383 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5384 return u32_to_fixed16(0);
5386 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5387 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5393 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5394 int width, const struct drm_format_info *format,
5395 u64 modifier, unsigned int rotation,
5396 u32 plane_pixel_rate, struct skl_wm_params *wp,
5399 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5400 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5403 /* only planar format has two planes */
5404 if (color_plane == 1 &&
5405 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5406 drm_dbg_kms(&dev_priv->drm,
5407 "Non planar format have single plane\n");
5411 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5412 modifier == I915_FORMAT_MOD_Yf_TILED ||
5413 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5414 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5415 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5416 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5417 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5418 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5421 if (color_plane == 1 && wp->is_planar)
5424 wp->cpp = format->cpp[color_plane];
5425 wp->plane_pixel_rate = plane_pixel_rate;
5427 if (DISPLAY_VER(dev_priv) >= 11 &&
5428 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5429 wp->dbuf_block_size = 256;
5431 wp->dbuf_block_size = 512;
5433 if (drm_rotation_90_or_270(rotation)) {
5436 wp->y_min_scanlines = 16;
5439 wp->y_min_scanlines = 8;
5442 wp->y_min_scanlines = 4;
5445 MISSING_CASE(wp->cpp);
5449 wp->y_min_scanlines = 4;
5452 if (skl_needs_memory_bw_wa(dev_priv))
5453 wp->y_min_scanlines *= 2;
5455 wp->plane_bytes_per_line = wp->width * wp->cpp;
5457 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5458 wp->y_min_scanlines,
5459 wp->dbuf_block_size);
5461 if (DISPLAY_VER(dev_priv) >= 10)
5464 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5465 wp->y_min_scanlines);
5467 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5468 wp->dbuf_block_size);
5470 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5473 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5476 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5477 wp->plane_blocks_per_line);
5479 wp->linetime_us = fixed16_to_u32_round_up(
5480 intel_get_linetime_us(crtc_state));
5486 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5487 const struct intel_plane_state *plane_state,
5488 struct skl_wm_params *wp, int color_plane)
5490 const struct drm_framebuffer *fb = plane_state->hw.fb;
5494 * Src coordinates are already rotated by 270 degrees for
5495 * the 90/270 degree plane rotation cases (to match the
5496 * GTT mapping), hence no need to account for rotation here.
5498 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5500 return skl_compute_wm_params(crtc_state, width,
5501 fb->format, fb->modifier,
5502 plane_state->hw.rotation,
5503 intel_plane_pixel_rate(crtc_state, plane_state),
5507 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5509 if (DISPLAY_VER(dev_priv) >= 10)
5512 /* The number of lines are ignored for the level 0 watermark. */
5516 static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5518 if (DISPLAY_VER(dev_priv) >= 13)
5524 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5525 struct intel_plane *plane,
5527 unsigned int latency,
5528 const struct skl_wm_params *wp,
5529 const struct skl_wm_level *result_prev,
5530 struct skl_wm_level *result /* out */)
5532 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5533 uint_fixed_16_16_t method1, method2;
5534 uint_fixed_16_16_t selected_result;
5535 u32 blocks, lines, min_ddb_alloc = 0;
5538 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
5540 result->min_ddb_alloc = U16_MAX;
5545 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5546 * Display WA #1141: kbl,cfl
5548 if ((IS_KABYLAKE(dev_priv) ||
5549 IS_COFFEELAKE(dev_priv) ||
5550 IS_COMETLAKE(dev_priv)) &&
5551 dev_priv->ipc_enabled)
5554 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5557 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5558 wp->cpp, latency, wp->dbuf_block_size);
5559 method2 = skl_wm_method2(wp->plane_pixel_rate,
5560 crtc_state->hw.pipe_mode.crtc_htotal,
5562 wp->plane_blocks_per_line);
5565 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5567 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5568 wp->dbuf_block_size < 1) &&
5569 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5570 selected_result = method2;
5571 } else if (latency >= wp->linetime_us) {
5572 if (DISPLAY_VER(dev_priv) == 9)
5573 selected_result = min_fixed16(method1, method2);
5575 selected_result = method2;
5577 selected_result = method1;
5581 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5582 lines = div_round_up_fixed16(selected_result,
5583 wp->plane_blocks_per_line);
5585 if (DISPLAY_VER(dev_priv) == 9) {
5586 /* Display WA #1125: skl,bxt,kbl */
5587 if (level == 0 && wp->rc_surface)
5588 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5590 /* Display WA #1126: skl,bxt,kbl */
5591 if (level >= 1 && level <= 7) {
5593 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5594 lines += wp->y_min_scanlines;
5600 * Make sure result blocks for higher latency levels are
5601 * atleast as high as level below the current level.
5602 * Assumption in DDB algorithm optimization for special
5603 * cases. Also covers Display WA #1125 for RC.
5605 if (result_prev->blocks > blocks)
5606 blocks = result_prev->blocks;
5610 if (DISPLAY_VER(dev_priv) >= 11) {
5614 if (lines % wp->y_min_scanlines == 0)
5615 extra_lines = wp->y_min_scanlines;
5617 extra_lines = wp->y_min_scanlines * 2 -
5618 lines % wp->y_min_scanlines;
5620 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5621 wp->plane_blocks_per_line);
5623 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5627 if (!skl_wm_has_lines(dev_priv, level))
5630 if (lines > skl_wm_max_lines(dev_priv)) {
5632 result->min_ddb_alloc = U16_MAX;
5637 * If lines is valid, assume we can use this watermark level
5638 * for now. We'll come back and disable it after we calculate the
5639 * DDB allocation if it turns out we don't actually have enough
5640 * blocks to satisfy it.
5642 result->blocks = blocks;
5643 result->lines = lines;
5644 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5645 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5646 result->enable = true;
5648 if (DISPLAY_VER(dev_priv) < 12)
5649 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5653 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5654 struct intel_plane *plane,
5655 const struct skl_wm_params *wm_params,
5656 struct skl_wm_level *levels)
5658 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5659 int level, max_level = ilk_wm_max_level(dev_priv);
5660 struct skl_wm_level *result_prev = &levels[0];
5662 for (level = 0; level <= max_level; level++) {
5663 struct skl_wm_level *result = &levels[level];
5664 unsigned int latency = dev_priv->wm.skl_latency[level];
5666 skl_compute_plane_wm(crtc_state, plane, level, latency,
5667 wm_params, result_prev, result);
5669 result_prev = result;
5673 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5674 struct intel_plane *plane,
5675 const struct skl_wm_params *wm_params,
5676 struct skl_plane_wm *plane_wm)
5678 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5679 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5680 struct skl_wm_level *levels = plane_wm->wm;
5681 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5683 skl_compute_plane_wm(crtc_state, plane, 0, latency,
5684 wm_params, &levels[0],
5688 static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5689 struct skl_wm_level *trans_wm,
5690 const struct skl_wm_level *wm0,
5691 const struct skl_wm_params *wp)
5693 u16 trans_min, trans_amount, trans_y_tile_min;
5694 u16 wm0_blocks, trans_offset, blocks;
5696 /* Transition WM don't make any sense if ipc is disabled */
5697 if (!dev_priv->ipc_enabled)
5701 * WaDisableTWM:skl,kbl,cfl,bxt
5702 * Transition WM are not recommended by HW team for GEN9
5704 if (DISPLAY_VER(dev_priv) == 9)
5707 if (DISPLAY_VER(dev_priv) >= 11)
5712 /* Display WA #1140: glk,cnl */
5713 if (DISPLAY_VER(dev_priv) == 10)
5716 trans_amount = 10; /* This is configurable amount */
5718 trans_offset = trans_min + trans_amount;
5721 * The spec asks for Selected Result Blocks for wm0 (the real value),
5722 * not Result Blocks (the integer value). Pay attention to the capital
5723 * letters. The value wm_l0->blocks is actually Result Blocks, but
5724 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5725 * and since we later will have to get the ceiling of the sum in the
5726 * transition watermarks calculation, we can just pretend Selected
5727 * Result Blocks is Result Blocks minus 1 and it should work for the
5728 * current platforms.
5730 wm0_blocks = wm0->blocks - 1;
5734 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5735 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5737 blocks = wm0_blocks + trans_offset;
5742 * Just assume we can enable the transition watermark. After
5743 * computing the DDB we'll come back and disable it if that
5744 * assumption turns out to be false.
5746 trans_wm->blocks = blocks;
5747 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5748 trans_wm->enable = true;
5751 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5752 const struct intel_plane_state *plane_state,
5753 struct intel_plane *plane, int color_plane)
5755 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5756 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5757 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5758 struct skl_wm_params wm_params;
5761 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5762 &wm_params, color_plane);
5766 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
5768 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5769 &wm->wm[0], &wm_params);
5771 if (DISPLAY_VER(dev_priv) >= 12) {
5772 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
5774 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5775 &wm->sagv.wm0, &wm_params);
5781 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5782 const struct intel_plane_state *plane_state,
5783 struct intel_plane *plane)
5785 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5786 struct skl_wm_params wm_params;
5789 wm->is_planar = true;
5791 /* uv plane watermarks must also be validated for NV12/Planar */
5792 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5797 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
5802 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5803 const struct intel_plane_state *plane_state)
5805 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5806 enum plane_id plane_id = plane->id;
5807 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5808 const struct drm_framebuffer *fb = plane_state->hw.fb;
5811 memset(wm, 0, sizeof(*wm));
5813 if (!intel_wm_plane_visible(crtc_state, plane_state))
5816 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5821 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5822 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5831 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5832 const struct intel_plane_state *plane_state)
5834 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5835 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5836 enum plane_id plane_id = plane->id;
5837 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5840 /* Watermarks calculated in master */
5841 if (plane_state->planar_slave)
5844 memset(wm, 0, sizeof(*wm));
5846 if (plane_state->planar_linked_plane) {
5847 const struct drm_framebuffer *fb = plane_state->hw.fb;
5849 drm_WARN_ON(&dev_priv->drm,
5850 !intel_wm_plane_visible(crtc_state, plane_state));
5851 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5852 fb->format->num_planes == 1);
5854 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5855 plane_state->planar_linked_plane, 0);
5859 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5863 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5864 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5873 static int skl_build_pipe_wm(struct intel_atomic_state *state,
5874 struct intel_crtc *crtc)
5876 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5877 struct intel_crtc_state *crtc_state =
5878 intel_atomic_get_new_crtc_state(state, crtc);
5879 const struct intel_plane_state *plane_state;
5880 struct intel_plane *plane;
5883 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5885 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5886 * instead but we don't populate that correctly for NV12 Y
5887 * planes so for now hack this.
5889 if (plane->pipe != crtc->pipe)
5892 if (DISPLAY_VER(dev_priv) >= 11)
5893 ret = icl_build_plane_wm(crtc_state, plane_state);
5895 ret = skl_build_plane_wm(crtc_state, plane_state);
5900 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5905 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5907 const struct skl_ddb_entry *entry)
5910 intel_de_write_fw(dev_priv, reg,
5911 PLANE_BUF_END(entry->end - 1) |
5912 PLANE_BUF_START(entry->start));
5914 intel_de_write_fw(dev_priv, reg, 0);
5917 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5919 const struct skl_wm_level *level)
5925 if (level->ignore_lines)
5926 val |= PLANE_WM_IGNORE_LINES;
5927 val |= level->blocks;
5928 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5930 intel_de_write_fw(dev_priv, reg, val);
5933 void skl_write_plane_wm(struct intel_plane *plane,
5934 const struct intel_crtc_state *crtc_state)
5936 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5937 int level, max_level = ilk_wm_max_level(dev_priv);
5938 enum plane_id plane_id = plane->id;
5939 enum pipe pipe = plane->pipe;
5940 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5941 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5942 const struct skl_ddb_entry *ddb_y =
5943 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5944 const struct skl_ddb_entry *ddb_uv =
5945 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5947 for (level = 0; level <= max_level; level++)
5948 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5949 skl_plane_wm_level(pipe_wm, plane_id, level));
5951 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5952 skl_plane_trans_wm(pipe_wm, plane_id));
5954 if (HAS_HW_SAGV_WM(dev_priv)) {
5955 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5957 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5958 &wm->sagv.trans_wm);
5961 if (DISPLAY_VER(dev_priv) >= 11) {
5962 skl_ddb_entry_write(dev_priv,
5963 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5968 swap(ddb_y, ddb_uv);
5970 skl_ddb_entry_write(dev_priv,
5971 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5972 skl_ddb_entry_write(dev_priv,
5973 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5976 void skl_write_cursor_wm(struct intel_plane *plane,
5977 const struct intel_crtc_state *crtc_state)
5979 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5980 int level, max_level = ilk_wm_max_level(dev_priv);
5981 enum plane_id plane_id = plane->id;
5982 enum pipe pipe = plane->pipe;
5983 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5984 const struct skl_ddb_entry *ddb =
5985 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5987 for (level = 0; level <= max_level; level++)
5988 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5989 skl_plane_wm_level(pipe_wm, plane_id, level));
5991 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5992 skl_plane_trans_wm(pipe_wm, plane_id));
5994 if (HAS_HW_SAGV_WM(dev_priv)) {
5995 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5997 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
5999 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
6000 &wm->sagv.trans_wm);
6003 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
6006 bool skl_wm_level_equals(const struct skl_wm_level *l1,
6007 const struct skl_wm_level *l2)
6009 return l1->enable == l2->enable &&
6010 l1->ignore_lines == l2->ignore_lines &&
6011 l1->lines == l2->lines &&
6012 l1->blocks == l2->blocks;
6015 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
6016 const struct skl_plane_wm *wm1,
6017 const struct skl_plane_wm *wm2)
6019 int level, max_level = ilk_wm_max_level(dev_priv);
6021 for (level = 0; level <= max_level; level++) {
6023 * We don't check uv_wm as the hardware doesn't actually
6024 * use it. It only gets used for calculating the required
6027 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
6031 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
6032 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
6033 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
6036 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
6037 const struct skl_ddb_entry *b)
6039 return a->start < b->end && b->start < a->end;
6042 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
6043 const struct skl_ddb_entry *b)
6045 if (a->end && b->end) {
6046 a->start = min(a->start, b->start);
6047 a->end = max(a->end, b->end);
6048 } else if (b->end) {
6049 a->start = b->start;
6054 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
6055 const struct skl_ddb_entry *entries,
6056 int num_entries, int ignore_idx)
6060 for (i = 0; i < num_entries; i++) {
6061 if (i != ignore_idx &&
6062 skl_ddb_entries_overlap(ddb, &entries[i]))
6070 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
6071 struct intel_crtc_state *new_crtc_state)
6073 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
6074 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6075 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6076 struct intel_plane *plane;
6078 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6079 struct intel_plane_state *plane_state;
6080 enum plane_id plane_id = plane->id;
6082 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
6083 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
6084 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
6085 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
6088 plane_state = intel_atomic_get_plane_state(state, plane);
6089 if (IS_ERR(plane_state))
6090 return PTR_ERR(plane_state);
6092 new_crtc_state->update_planes |= BIT(plane_id);
6098 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
6100 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
6105 * FIXME: For now we always enable slice S1 as per
6106 * the Bspec display initialization sequence.
6108 enabled_slices = BIT(DBUF_S1);
6110 for_each_pipe(dev_priv, pipe)
6111 enabled_slices |= dbuf_state->slices[pipe];
6113 return enabled_slices;
6117 skl_compute_ddb(struct intel_atomic_state *state)
6119 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6120 const struct intel_dbuf_state *old_dbuf_state;
6121 struct intel_dbuf_state *new_dbuf_state = NULL;
6122 const struct intel_crtc_state *old_crtc_state;
6123 struct intel_crtc_state *new_crtc_state;
6124 struct intel_crtc *crtc;
6127 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6128 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6129 if (IS_ERR(new_dbuf_state))
6130 return PTR_ERR(new_dbuf_state);
6132 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6136 if (!new_dbuf_state)
6139 new_dbuf_state->active_pipes =
6140 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6142 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
6143 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6148 if (IS_ALDERLAKE_P(dev_priv))
6149 new_dbuf_state->joined_mbus =
6150 adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6152 for_each_intel_crtc(&dev_priv->drm, crtc) {
6153 enum pipe pipe = crtc->pipe;
6155 new_dbuf_state->slices[pipe] =
6156 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
6157 new_dbuf_state->joined_mbus);
6159 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
6162 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6167 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6169 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6170 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6171 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6175 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6176 /* TODO: Implement vblank synchronized MBUS joining changes */
6177 ret = intel_modeset_all_pipes(state);
6182 drm_dbg_kms(&dev_priv->drm,
6183 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6184 old_dbuf_state->enabled_slices,
6185 new_dbuf_state->enabled_slices,
6186 INTEL_INFO(dev_priv)->dbuf.slice_mask,
6187 yesno(old_dbuf_state->joined_mbus),
6188 yesno(new_dbuf_state->joined_mbus));
6191 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6192 enum pipe pipe = crtc->pipe;
6194 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6196 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6199 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6204 for_each_intel_crtc(&dev_priv->drm, crtc) {
6205 ret = skl_crtc_allocate_ddb(state, crtc);
6210 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6211 new_crtc_state, i) {
6212 ret = skl_crtc_allocate_plane_ddb(state, crtc);
6216 ret = skl_ddb_add_affected_planes(old_crtc_state,
6225 static char enast(bool enable)
6227 return enable ? '*' : ' ';
6231 skl_print_wm_changes(struct intel_atomic_state *state)
6233 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6234 const struct intel_crtc_state *old_crtc_state;
6235 const struct intel_crtc_state *new_crtc_state;
6236 struct intel_plane *plane;
6237 struct intel_crtc *crtc;
6240 if (!drm_debug_enabled(DRM_UT_KMS))
6243 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6244 new_crtc_state, i) {
6245 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6247 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6248 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6250 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6251 enum plane_id plane_id = plane->id;
6252 const struct skl_ddb_entry *old, *new;
6254 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
6255 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
6257 if (skl_ddb_entry_equal(old, new))
6260 drm_dbg_kms(&dev_priv->drm,
6261 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6262 plane->base.base.id, plane->base.name,
6263 old->start, old->end, new->start, new->end,
6264 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6267 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6268 enum plane_id plane_id = plane->id;
6269 const struct skl_plane_wm *old_wm, *new_wm;
6271 old_wm = &old_pipe_wm->planes[plane_id];
6272 new_wm = &new_pipe_wm->planes[plane_id];
6274 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6277 drm_dbg_kms(&dev_priv->drm,
6278 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6279 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6280 plane->base.base.id, plane->base.name,
6281 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6282 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6283 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6284 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6285 enast(old_wm->trans_wm.enable),
6286 enast(old_wm->sagv.wm0.enable),
6287 enast(old_wm->sagv.trans_wm.enable),
6288 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6289 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6290 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6291 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6292 enast(new_wm->trans_wm.enable),
6293 enast(new_wm->sagv.wm0.enable),
6294 enast(new_wm->sagv.trans_wm.enable));
6296 drm_dbg_kms(&dev_priv->drm,
6297 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6298 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6299 plane->base.base.id, plane->base.name,
6300 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6301 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6302 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6303 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6304 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6305 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6306 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6307 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6308 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6309 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6310 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6311 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6312 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6313 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6314 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6315 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6316 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6317 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6318 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6319 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6320 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6321 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6323 drm_dbg_kms(&dev_priv->drm,
6324 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6325 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6326 plane->base.base.id, plane->base.name,
6327 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6328 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6329 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6330 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6331 old_wm->trans_wm.blocks,
6332 old_wm->sagv.wm0.blocks,
6333 old_wm->sagv.trans_wm.blocks,
6334 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6335 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6336 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6337 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6338 new_wm->trans_wm.blocks,
6339 new_wm->sagv.wm0.blocks,
6340 new_wm->sagv.trans_wm.blocks);
6342 drm_dbg_kms(&dev_priv->drm,
6343 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6344 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6345 plane->base.base.id, plane->base.name,
6346 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6347 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6348 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6349 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6350 old_wm->trans_wm.min_ddb_alloc,
6351 old_wm->sagv.wm0.min_ddb_alloc,
6352 old_wm->sagv.trans_wm.min_ddb_alloc,
6353 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6354 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6355 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6356 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6357 new_wm->trans_wm.min_ddb_alloc,
6358 new_wm->sagv.wm0.min_ddb_alloc,
6359 new_wm->sagv.trans_wm.min_ddb_alloc);
6364 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6365 const struct skl_pipe_wm *old_pipe_wm,
6366 const struct skl_pipe_wm *new_pipe_wm)
6368 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6369 int level, max_level = ilk_wm_max_level(i915);
6371 for (level = 0; level <= max_level; level++) {
6373 * We don't check uv_wm as the hardware doesn't actually
6374 * use it. It only gets used for calculating the required
6377 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6378 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6382 if (HAS_HW_SAGV_WM(i915)) {
6383 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6384 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6386 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6387 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6391 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6392 skl_plane_trans_wm(new_pipe_wm, plane->id));
6396 * To make sure the cursor watermark registers are always consistent
6397 * with our computed state the following scenario needs special
6401 * 2. move cursor entirely offscreen
6404 * Step 2. does call .disable_plane() but does not zero the watermarks
6405 * (since we consider an offscreen cursor still active for the purposes
6406 * of watermarks). Step 3. would not normally call .disable_plane()
6407 * because the actual plane visibility isn't changing, and we don't
6408 * deallocate the cursor ddb until the pipe gets disabled. So we must
6409 * force step 3. to call .disable_plane() to update the watermark
6410 * registers properly.
6412 * Other planes do not suffer from this issues as their watermarks are
6413 * calculated based on the actual plane visibility. The only time this
6414 * can trigger for the other planes is during the initial readout as the
6415 * default value of the watermarks registers is not zero.
6417 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6418 struct intel_crtc *crtc)
6420 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6421 const struct intel_crtc_state *old_crtc_state =
6422 intel_atomic_get_old_crtc_state(state, crtc);
6423 struct intel_crtc_state *new_crtc_state =
6424 intel_atomic_get_new_crtc_state(state, crtc);
6425 struct intel_plane *plane;
6427 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6428 struct intel_plane_state *plane_state;
6429 enum plane_id plane_id = plane->id;
6432 * Force a full wm update for every plane on modeset.
6433 * Required because the reset value of the wm registers
6434 * is non-zero, whereas we want all disabled planes to
6435 * have zero watermarks. So if we turn off the relevant
6436 * power well the hardware state will go out of sync
6437 * with the software state.
6439 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6440 skl_plane_selected_wm_equals(plane,
6441 &old_crtc_state->wm.skl.optimal,
6442 &new_crtc_state->wm.skl.optimal))
6445 plane_state = intel_atomic_get_plane_state(state, plane);
6446 if (IS_ERR(plane_state))
6447 return PTR_ERR(plane_state);
6449 new_crtc_state->update_planes |= BIT(plane_id);
6456 skl_compute_wm(struct intel_atomic_state *state)
6458 struct intel_crtc *crtc;
6459 struct intel_crtc_state *new_crtc_state;
6462 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6463 ret = skl_build_pipe_wm(state, crtc);
6468 ret = skl_compute_ddb(state);
6472 ret = intel_compute_sagv_mask(state);
6477 * skl_compute_ddb() will have adjusted the final watermarks
6478 * based on how much ddb is available. Now we can actually
6479 * check if the final watermarks changed.
6481 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6482 ret = skl_wm_add_affected_planes(state, crtc);
6487 skl_print_wm_changes(state);
6492 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6493 struct intel_wm_config *config)
6495 struct intel_crtc *crtc;
6497 /* Compute the currently _active_ config */
6498 for_each_intel_crtc(&dev_priv->drm, crtc) {
6499 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6501 if (!wm->pipe_enabled)
6504 config->sprites_enabled |= wm->sprites_enabled;
6505 config->sprites_scaled |= wm->sprites_scaled;
6506 config->num_pipes_active++;
6510 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6512 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6513 struct ilk_wm_maximums max;
6514 struct intel_wm_config config = {};
6515 struct ilk_wm_values results = {};
6516 enum intel_ddb_partitioning partitioning;
6518 ilk_compute_wm_config(dev_priv, &config);
6520 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6521 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6523 /* 5/6 split only in single pipe config on IVB+ */
6524 if (DISPLAY_VER(dev_priv) >= 7 &&
6525 config.num_pipes_active == 1 && config.sprites_enabled) {
6526 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6527 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6529 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6531 best_lp_wm = &lp_wm_1_2;
6534 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6535 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6537 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6539 ilk_write_wm_values(dev_priv, &results);
6542 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6543 struct intel_crtc *crtc)
6545 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6546 const struct intel_crtc_state *crtc_state =
6547 intel_atomic_get_new_crtc_state(state, crtc);
6549 mutex_lock(&dev_priv->wm.wm_mutex);
6550 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6551 ilk_program_watermarks(dev_priv);
6552 mutex_unlock(&dev_priv->wm.wm_mutex);
6555 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6556 struct intel_crtc *crtc)
6558 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6559 const struct intel_crtc_state *crtc_state =
6560 intel_atomic_get_new_crtc_state(state, crtc);
6562 if (!crtc_state->wm.need_postvbl_update)
6565 mutex_lock(&dev_priv->wm.wm_mutex);
6566 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6567 ilk_program_watermarks(dev_priv);
6568 mutex_unlock(&dev_priv->wm.wm_mutex);
6571 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6573 level->enable = val & PLANE_WM_EN;
6574 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6575 level->blocks = val & PLANE_WM_BLOCKS_MASK;
6576 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6579 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6580 struct skl_pipe_wm *out)
6582 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6583 enum pipe pipe = crtc->pipe;
6584 int level, max_level;
6585 enum plane_id plane_id;
6588 max_level = ilk_wm_max_level(dev_priv);
6590 for_each_plane_id_on_crtc(crtc, plane_id) {
6591 struct skl_plane_wm *wm = &out->planes[plane_id];
6593 for (level = 0; level <= max_level; level++) {
6594 if (plane_id != PLANE_CURSOR)
6595 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6597 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6599 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6602 if (plane_id != PLANE_CURSOR)
6603 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6605 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6607 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6609 if (HAS_HW_SAGV_WM(dev_priv)) {
6610 if (plane_id != PLANE_CURSOR)
6611 val = intel_uncore_read(&dev_priv->uncore,
6612 PLANE_WM_SAGV(pipe, plane_id));
6614 val = intel_uncore_read(&dev_priv->uncore,
6617 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6619 if (plane_id != PLANE_CURSOR)
6620 val = intel_uncore_read(&dev_priv->uncore,
6621 PLANE_WM_SAGV_TRANS(pipe, plane_id));
6623 val = intel_uncore_read(&dev_priv->uncore,
6624 CUR_WM_SAGV_TRANS(pipe));
6626 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6627 } else if (DISPLAY_VER(dev_priv) >= 12) {
6628 wm->sagv.wm0 = wm->wm[0];
6629 wm->sagv.trans_wm = wm->trans_wm;
6634 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6636 struct intel_dbuf_state *dbuf_state =
6637 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6638 struct intel_crtc *crtc;
6640 if (IS_ALDERLAKE_P(dev_priv))
6641 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6643 for_each_intel_crtc(&dev_priv->drm, crtc) {
6644 struct intel_crtc_state *crtc_state =
6645 to_intel_crtc_state(crtc->base.state);
6646 enum pipe pipe = crtc->pipe;
6647 unsigned int mbus_offset;
6648 enum plane_id plane_id;
6651 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6652 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6654 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6656 for_each_plane_id_on_crtc(crtc, plane_id) {
6657 struct skl_ddb_entry *ddb_y =
6658 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6659 struct skl_ddb_entry *ddb_uv =
6660 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6662 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6663 plane_id, ddb_y, ddb_uv);
6665 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6666 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6669 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6672 * Used for checking overlaps, so we need absolute
6673 * offsets instead of MBUS relative offsets.
6675 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6676 dbuf_state->joined_mbus);
6677 mbus_offset = mbus_ddb_offset(dev_priv, slices);
6678 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6679 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6681 /* The slices actually used by the planes on the pipe */
6682 dbuf_state->slices[pipe] =
6683 skl_ddb_dbuf_slice_mask(dev_priv, &crtc_state->wm.skl.ddb);
6685 drm_dbg_kms(&dev_priv->drm,
6686 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6687 crtc->base.base.id, crtc->base.name,
6688 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6689 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6690 yesno(dbuf_state->joined_mbus));
6693 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6696 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
6698 const struct intel_dbuf_state *dbuf_state =
6699 to_intel_dbuf_state(i915->dbuf.obj.state);
6700 struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
6701 struct intel_crtc *crtc;
6703 for_each_intel_crtc(&i915->drm, crtc) {
6704 const struct intel_crtc_state *crtc_state =
6705 to_intel_crtc_state(crtc->base.state);
6707 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
6710 for_each_intel_crtc(&i915->drm, crtc) {
6711 const struct intel_crtc_state *crtc_state =
6712 to_intel_crtc_state(crtc->base.state);
6715 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6716 dbuf_state->joined_mbus);
6717 if (dbuf_state->slices[crtc->pipe] & ~slices)
6720 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
6721 I915_MAX_PIPES, crtc->pipe))
6728 void skl_wm_sanitize(struct drm_i915_private *i915)
6730 struct intel_crtc *crtc;
6733 * On TGL/RKL (at least) the BIOS likes to assign the planes
6734 * to the wrong DBUF slices. This will cause an infinite loop
6735 * in skl_commit_modeset_enables() as it can't find a way to
6736 * transition between the old bogus DBUF layout to the new
6737 * proper DBUF layout without DBUF allocation overlaps between
6738 * the planes (which cannot be allowed or else the hardware
6739 * may hang). If we detect a bogus DBUF layout just turn off
6740 * all the planes so that skl_commit_modeset_enables() can
6741 * simply ignore them.
6743 if (!skl_dbuf_is_misconfigured(i915))
6746 drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
6748 for_each_intel_crtc(&i915->drm, crtc) {
6749 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6750 const struct intel_plane_state *plane_state =
6751 to_intel_plane_state(plane->base.state);
6752 struct intel_crtc_state *crtc_state =
6753 to_intel_crtc_state(crtc->base.state);
6755 if (plane_state->uapi.visible)
6756 intel_plane_disable_noatomic(crtc, plane);
6758 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
6760 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
6764 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6766 struct drm_device *dev = crtc->base.dev;
6767 struct drm_i915_private *dev_priv = to_i915(dev);
6768 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6769 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6770 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6771 enum pipe pipe = crtc->pipe;
6773 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6775 memset(active, 0, sizeof(*active));
6777 active->pipe_enabled = crtc->active;
6779 if (active->pipe_enabled) {
6780 u32 tmp = hw->wm_pipe[pipe];
6783 * For active pipes LP0 watermark is marked as
6784 * enabled, and LP1+ watermaks as disabled since
6785 * we can't really reverse compute them in case
6786 * multiple pipes are active.
6788 active->wm[0].enable = true;
6789 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
6790 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
6791 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
6793 int level, max_level = ilk_wm_max_level(dev_priv);
6796 * For inactive pipes, all watermark levels
6797 * should be marked as enabled but zeroed,
6798 * which is what we'd compute them to.
6800 for (level = 0; level <= max_level; level++)
6801 active->wm[level].enable = true;
6804 crtc->wm.active.ilk = *active;
6807 #define _FW_WM(value, plane) \
6808 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6809 #define _FW_WM_VLV(value, plane) \
6810 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6812 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6813 struct g4x_wm_values *wm)
6817 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6818 wm->sr.plane = _FW_WM(tmp, SR);
6819 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6820 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6821 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6823 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6824 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6825 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6826 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6827 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6828 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6829 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6831 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6832 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6833 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6834 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6835 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6838 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6839 struct vlv_wm_values *wm)
6844 for_each_pipe(dev_priv, pipe) {
6845 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6847 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6848 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6849 wm->ddl[pipe].plane[PLANE_CURSOR] =
6850 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6851 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6852 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6853 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6854 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6857 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6858 wm->sr.plane = _FW_WM(tmp, SR);
6859 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6860 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6861 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6863 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6864 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6865 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6866 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6868 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6869 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6871 if (IS_CHERRYVIEW(dev_priv)) {
6872 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6873 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6874 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6876 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6877 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6878 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6880 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6881 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6882 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6884 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6885 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6886 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6887 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6888 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6889 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6890 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6891 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6892 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6893 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6894 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6896 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6897 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6898 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6900 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6901 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6902 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6903 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6904 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6905 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6906 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6907 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6914 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6916 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6917 struct intel_crtc *crtc;
6919 g4x_read_wm_values(dev_priv, wm);
6921 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6923 for_each_intel_crtc(&dev_priv->drm, crtc) {
6924 struct intel_crtc_state *crtc_state =
6925 to_intel_crtc_state(crtc->base.state);
6926 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6927 struct g4x_pipe_wm *raw;
6928 enum pipe pipe = crtc->pipe;
6929 enum plane_id plane_id;
6930 int level, max_level;
6932 active->cxsr = wm->cxsr;
6933 active->hpll_en = wm->hpll_en;
6934 active->fbc_en = wm->fbc_en;
6936 active->sr = wm->sr;
6937 active->hpll = wm->hpll;
6939 for_each_plane_id_on_crtc(crtc, plane_id) {
6940 active->wm.plane[plane_id] =
6941 wm->pipe[pipe].plane[plane_id];
6944 if (wm->cxsr && wm->hpll_en)
6945 max_level = G4X_WM_LEVEL_HPLL;
6947 max_level = G4X_WM_LEVEL_SR;
6949 max_level = G4X_WM_LEVEL_NORMAL;
6951 level = G4X_WM_LEVEL_NORMAL;
6952 raw = &crtc_state->wm.g4x.raw[level];
6953 for_each_plane_id_on_crtc(crtc, plane_id)
6954 raw->plane[plane_id] = active->wm.plane[plane_id];
6956 level = G4X_WM_LEVEL_SR;
6957 if (level > max_level)
6960 raw = &crtc_state->wm.g4x.raw[level];
6961 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6962 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6963 raw->plane[PLANE_SPRITE0] = 0;
6964 raw->fbc = active->sr.fbc;
6966 level = G4X_WM_LEVEL_HPLL;
6967 if (level > max_level)
6970 raw = &crtc_state->wm.g4x.raw[level];
6971 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6972 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6973 raw->plane[PLANE_SPRITE0] = 0;
6974 raw->fbc = active->hpll.fbc;
6978 for_each_plane_id_on_crtc(crtc, plane_id)
6979 g4x_raw_plane_wm_set(crtc_state, level,
6980 plane_id, USHRT_MAX);
6981 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6983 crtc_state->wm.g4x.optimal = *active;
6984 crtc_state->wm.g4x.intermediate = *active;
6986 drm_dbg_kms(&dev_priv->drm,
6987 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6989 wm->pipe[pipe].plane[PLANE_PRIMARY],
6990 wm->pipe[pipe].plane[PLANE_CURSOR],
6991 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6994 drm_dbg_kms(&dev_priv->drm,
6995 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6996 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6997 drm_dbg_kms(&dev_priv->drm,
6998 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6999 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
7000 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
7001 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
7004 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
7006 struct intel_plane *plane;
7007 struct intel_crtc *crtc;
7009 mutex_lock(&dev_priv->wm.wm_mutex);
7011 for_each_intel_plane(&dev_priv->drm, plane) {
7012 struct intel_crtc *crtc =
7013 intel_crtc_for_pipe(dev_priv, plane->pipe);
7014 struct intel_crtc_state *crtc_state =
7015 to_intel_crtc_state(crtc->base.state);
7016 struct intel_plane_state *plane_state =
7017 to_intel_plane_state(plane->base.state);
7018 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
7019 enum plane_id plane_id = plane->id;
7022 if (plane_state->uapi.visible)
7025 for (level = 0; level < 3; level++) {
7026 struct g4x_pipe_wm *raw =
7027 &crtc_state->wm.g4x.raw[level];
7029 raw->plane[plane_id] = 0;
7030 wm_state->wm.plane[plane_id] = 0;
7033 if (plane_id == PLANE_PRIMARY) {
7034 for (level = 0; level < 3; level++) {
7035 struct g4x_pipe_wm *raw =
7036 &crtc_state->wm.g4x.raw[level];
7040 wm_state->sr.fbc = 0;
7041 wm_state->hpll.fbc = 0;
7042 wm_state->fbc_en = false;
7046 for_each_intel_crtc(&dev_priv->drm, crtc) {
7047 struct intel_crtc_state *crtc_state =
7048 to_intel_crtc_state(crtc->base.state);
7050 crtc_state->wm.g4x.intermediate =
7051 crtc_state->wm.g4x.optimal;
7052 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
7055 g4x_program_watermarks(dev_priv);
7057 mutex_unlock(&dev_priv->wm.wm_mutex);
7060 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
7062 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
7063 struct intel_crtc *crtc;
7066 vlv_read_wm_values(dev_priv, wm);
7068 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
7069 wm->level = VLV_WM_LEVEL_PM2;
7071 if (IS_CHERRYVIEW(dev_priv)) {
7072 vlv_punit_get(dev_priv);
7074 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
7075 if (val & DSP_MAXFIFO_PM5_ENABLE)
7076 wm->level = VLV_WM_LEVEL_PM5;
7079 * If DDR DVFS is disabled in the BIOS, Punit
7080 * will never ack the request. So if that happens
7081 * assume we don't have to enable/disable DDR DVFS
7082 * dynamically. To test that just set the REQ_ACK
7083 * bit to poke the Punit, but don't change the
7084 * HIGH/LOW bits so that we don't actually change
7085 * the current state.
7087 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7088 val |= FORCE_DDR_FREQ_REQ_ACK;
7089 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
7091 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
7092 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
7093 drm_dbg_kms(&dev_priv->drm,
7094 "Punit not acking DDR DVFS request, "
7095 "assuming DDR DVFS is disabled\n");
7096 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
7098 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7099 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
7100 wm->level = VLV_WM_LEVEL_DDR_DVFS;
7103 vlv_punit_put(dev_priv);
7106 for_each_intel_crtc(&dev_priv->drm, crtc) {
7107 struct intel_crtc_state *crtc_state =
7108 to_intel_crtc_state(crtc->base.state);
7109 struct vlv_wm_state *active = &crtc->wm.active.vlv;
7110 const struct vlv_fifo_state *fifo_state =
7111 &crtc_state->wm.vlv.fifo_state;
7112 enum pipe pipe = crtc->pipe;
7113 enum plane_id plane_id;
7116 vlv_get_fifo_size(crtc_state);
7118 active->num_levels = wm->level + 1;
7119 active->cxsr = wm->cxsr;
7121 for (level = 0; level < active->num_levels; level++) {
7122 struct g4x_pipe_wm *raw =
7123 &crtc_state->wm.vlv.raw[level];
7125 active->sr[level].plane = wm->sr.plane;
7126 active->sr[level].cursor = wm->sr.cursor;
7128 for_each_plane_id_on_crtc(crtc, plane_id) {
7129 active->wm[level].plane[plane_id] =
7130 wm->pipe[pipe].plane[plane_id];
7132 raw->plane[plane_id] =
7133 vlv_invert_wm_value(active->wm[level].plane[plane_id],
7134 fifo_state->plane[plane_id]);
7138 for_each_plane_id_on_crtc(crtc, plane_id)
7139 vlv_raw_plane_wm_set(crtc_state, level,
7140 plane_id, USHRT_MAX);
7141 vlv_invalidate_wms(crtc, active, level);
7143 crtc_state->wm.vlv.optimal = *active;
7144 crtc_state->wm.vlv.intermediate = *active;
7146 drm_dbg_kms(&dev_priv->drm,
7147 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
7149 wm->pipe[pipe].plane[PLANE_PRIMARY],
7150 wm->pipe[pipe].plane[PLANE_CURSOR],
7151 wm->pipe[pipe].plane[PLANE_SPRITE0],
7152 wm->pipe[pipe].plane[PLANE_SPRITE1]);
7155 drm_dbg_kms(&dev_priv->drm,
7156 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
7157 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
7160 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
7162 struct intel_plane *plane;
7163 struct intel_crtc *crtc;
7165 mutex_lock(&dev_priv->wm.wm_mutex);
7167 for_each_intel_plane(&dev_priv->drm, plane) {
7168 struct intel_crtc *crtc =
7169 intel_crtc_for_pipe(dev_priv, plane->pipe);
7170 struct intel_crtc_state *crtc_state =
7171 to_intel_crtc_state(crtc->base.state);
7172 struct intel_plane_state *plane_state =
7173 to_intel_plane_state(plane->base.state);
7174 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
7175 const struct vlv_fifo_state *fifo_state =
7176 &crtc_state->wm.vlv.fifo_state;
7177 enum plane_id plane_id = plane->id;
7180 if (plane_state->uapi.visible)
7183 for (level = 0; level < wm_state->num_levels; level++) {
7184 struct g4x_pipe_wm *raw =
7185 &crtc_state->wm.vlv.raw[level];
7187 raw->plane[plane_id] = 0;
7189 wm_state->wm[level].plane[plane_id] =
7190 vlv_invert_wm_value(raw->plane[plane_id],
7191 fifo_state->plane[plane_id]);
7195 for_each_intel_crtc(&dev_priv->drm, crtc) {
7196 struct intel_crtc_state *crtc_state =
7197 to_intel_crtc_state(crtc->base.state);
7199 crtc_state->wm.vlv.intermediate =
7200 crtc_state->wm.vlv.optimal;
7201 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
7204 vlv_program_watermarks(dev_priv);
7206 mutex_unlock(&dev_priv->wm.wm_mutex);
7210 * FIXME should probably kill this and improve
7211 * the real watermark readout/sanitation instead
7213 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
7215 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM_LP_ENABLE);
7216 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM_LP_ENABLE);
7217 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM_LP_ENABLE);
7220 * Don't touch WM_LP_SPRITE_ENABLE here.
7221 * Doing so could cause underruns.
7225 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
7227 struct ilk_wm_values *hw = &dev_priv->wm.hw;
7228 struct intel_crtc *crtc;
7230 ilk_init_lp_watermarks(dev_priv);
7232 for_each_intel_crtc(&dev_priv->drm, crtc)
7233 ilk_pipe_wm_get_hw_state(crtc);
7235 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7236 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7237 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7239 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7240 if (DISPLAY_VER(dev_priv) >= 7) {
7241 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7242 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7245 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7246 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7247 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7248 else if (IS_IVYBRIDGE(dev_priv))
7249 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7250 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7253 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7256 void intel_enable_ipc(struct drm_i915_private *dev_priv)
7260 if (!HAS_IPC(dev_priv))
7263 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7265 if (dev_priv->ipc_enabled)
7266 val |= DISP_IPC_ENABLE;
7268 val &= ~DISP_IPC_ENABLE;
7270 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7273 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7275 /* Display WA #0477 WaDisableIPC: skl */
7276 if (IS_SKYLAKE(dev_priv))
7279 /* Display WA #1141: SKL:all KBL:all CFL */
7280 if (IS_KABYLAKE(dev_priv) ||
7281 IS_COFFEELAKE(dev_priv) ||
7282 IS_COMETLAKE(dev_priv))
7283 return dev_priv->dram_info.symmetric_memory;
7288 void intel_init_ipc(struct drm_i915_private *dev_priv)
7290 if (!HAS_IPC(dev_priv))
7293 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7295 intel_enable_ipc(dev_priv);
7298 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7301 * On Ibex Peak and Cougar Point, we need to disable clock
7302 * gating for the panel power sequencer or it will fail to
7303 * start up when no ports are active.
7305 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7308 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7312 for_each_pipe(dev_priv, pipe) {
7313 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7314 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7315 DISP_TRICKLE_FEED_DISABLE);
7317 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7318 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7322 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7324 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7328 * WaFbcDisableDpfcClockGating:ilk
7330 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7331 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7332 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7334 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7335 MARIUNIT_CLOCK_GATE_DISABLE |
7336 SVSMUNIT_CLOCK_GATE_DISABLE);
7337 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7338 VFMUNIT_CLOCK_GATE_DISABLE);
7341 * According to the spec the following bits should be set in
7342 * order to enable memory self-refresh
7343 * The bit 22/21 of 0x42004
7344 * The bit 5 of 0x42020
7345 * The bit 15 of 0x45000
7347 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7348 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7349 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7350 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7351 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7352 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7356 * Based on the document from hardware guys the following bits
7357 * should be set unconditionally in order to enable FBC.
7358 * The bit 22 of 0x42000
7359 * The bit 22 of 0x42004
7360 * The bit 7,8,9 of 0x42020.
7362 if (IS_IRONLAKE_M(dev_priv)) {
7363 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
7364 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7365 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7367 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7368 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7372 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7374 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7375 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7376 ILK_ELPIN_409_SELECT);
7378 g4x_disable_trickle_feed(dev_priv);
7380 ibx_init_clock_gating(dev_priv);
7383 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7389 * On Ibex Peak and Cougar Point, we need to disable clock
7390 * gating for the panel power sequencer or it will fail to
7391 * start up when no ports are active.
7393 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7394 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7395 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7396 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7397 DPLS_EDP_PPS_FIX_DIS);
7398 /* The below fixes the weird display corruption, a few pixels shifted
7399 * downward, on (only) LVDS of some HP laptops with IVY.
7401 for_each_pipe(dev_priv, pipe) {
7402 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7403 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7404 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7405 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7406 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7407 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7408 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7409 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7411 /* WADP0ClockGatingDisable */
7412 for_each_pipe(dev_priv, pipe) {
7413 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7414 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7418 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7422 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7423 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
7424 drm_dbg_kms(&dev_priv->drm,
7425 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7429 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7431 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7433 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7435 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7436 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7437 ILK_ELPIN_409_SELECT);
7439 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7440 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7441 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7442 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7444 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7445 * gating disable must be set. Failure to set it results in
7446 * flickering pixels due to Z write ordering failures after
7447 * some amount of runtime in the Mesa "fire" demo, and Unigine
7448 * Sanctuary and Tropics, and apparently anything else with
7449 * alpha test or pixel discard.
7451 * According to the spec, bit 11 (RCCUNIT) must also be set,
7452 * but we didn't debug actual testcases to find it out.
7454 * WaDisableRCCUnitClockGating:snb
7455 * WaDisableRCPBUnitClockGating:snb
7457 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7458 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7459 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7462 * According to the spec the following bits should be
7463 * set in order to enable memory self-refresh and fbc:
7464 * The bit21 and bit22 of 0x42000
7465 * The bit21 and bit22 of 0x42004
7466 * The bit5 and bit7 of 0x42020
7467 * The bit14 of 0x70180
7468 * The bit14 of 0x71180
7470 * WaFbcAsynchFlipDisableFbcQueue:snb
7472 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7473 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7474 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7475 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7476 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7477 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7478 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7479 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7480 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7481 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7483 g4x_disable_trickle_feed(dev_priv);
7485 cpt_init_clock_gating(dev_priv);
7487 gen6_check_mch_setup(dev_priv);
7490 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7493 * TODO: this bit should only be enabled when really needed, then
7494 * disabled when not needed anymore in order to save power.
7496 if (HAS_PCH_LPT_LP(dev_priv))
7497 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7498 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7499 PCH_LP_PARTITION_LEVEL_DISABLE);
7501 /* WADPOClockGatingDisable:hsw */
7502 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7503 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7504 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7507 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7509 if (HAS_PCH_LPT_LP(dev_priv)) {
7510 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7512 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7513 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7517 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7518 int general_prio_credits,
7519 int high_prio_credits)
7524 /* WaTempDisableDOPClkGating:bdw */
7525 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7526 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7528 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7529 val &= ~L3_PRIO_CREDITS_MASK;
7530 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7531 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7532 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7535 * Wait at least 100 clocks before re-enabling clock gating.
7536 * See the definition of L3SQCREG1 in BSpec.
7538 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7540 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7543 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7545 /* Wa_1409120013:icl,ehl */
7546 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7547 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7549 /*Wa_14010594013:icl, ehl */
7550 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7551 0, ICL_DELAY_PMRSP);
7554 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7556 /* Wa_1409120013:tgl,rkl,adl-s,dg1,dg2 */
7557 if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) ||
7558 IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_DG2(dev_priv))
7559 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7560 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7562 /* Wa_1409825376:tgl (pre-prod)*/
7563 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
7564 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7565 TGL_VRH_GATING_DIS);
7567 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
7568 if (DISPLAY_VER(dev_priv) == 12)
7569 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7570 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7573 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7575 gen12lp_init_clock_gating(dev_priv);
7577 /* Wa_22011091694:adlp */
7578 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7581 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7583 gen12lp_init_clock_gating(dev_priv);
7585 /* Wa_1409836686:dg1[a0] */
7586 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7587 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7591 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
7593 /* Wa_22010146351:xehpsdv */
7594 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7595 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
7598 static void dg2_init_clock_gating(struct drm_i915_private *i915)
7600 /* Wa_22010954014:dg2_g10 */
7601 if (IS_DG2_G10(i915))
7602 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7606 * Wa_14010733611:dg2_g10
7607 * Wa_22010146351:dg2_g10
7609 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
7610 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7611 SGR_DIS | SGGI_DIS);
7614 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7616 if (!HAS_PCH_CNP(dev_priv))
7619 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7620 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7621 CNP_PWM_CGE_GATING_DISABLE);
7624 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7626 cnp_init_clock_gating(dev_priv);
7627 gen9_init_clock_gating(dev_priv);
7629 /* WAC6entrylatency:cfl */
7630 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7631 FBC_LLC_FULLY_OPEN);
7634 * WaFbcTurnOffFbcWatermark:cfl
7635 * Display WA #0562: cfl
7637 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7641 * WaFbcNukeOnHostModify:cfl
7642 * Display WA #0873: cfl
7644 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7645 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7646 DPFC_NUKE_ON_ANY_MODIFICATION);
7649 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7651 gen9_init_clock_gating(dev_priv);
7653 /* WAC6entrylatency:kbl */
7654 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7655 FBC_LLC_FULLY_OPEN);
7657 /* WaDisableSDEUnitClockGating:kbl */
7658 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7659 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7660 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7662 /* WaDisableGamClockGating:kbl */
7663 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7664 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7665 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7668 * WaFbcTurnOffFbcWatermark:kbl
7669 * Display WA #0562: kbl
7671 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7675 * WaFbcNukeOnHostModify:kbl
7676 * Display WA #0873: kbl
7678 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7679 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7680 DPFC_NUKE_ON_ANY_MODIFICATION);
7683 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7685 gen9_init_clock_gating(dev_priv);
7687 /* WaDisableDopClockGating:skl */
7688 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7689 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7691 /* WAC6entrylatency:skl */
7692 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7693 FBC_LLC_FULLY_OPEN);
7696 * WaFbcTurnOffFbcWatermark:skl
7697 * Display WA #0562: skl
7699 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7703 * WaFbcNukeOnHostModify:skl
7704 * Display WA #0873: skl
7706 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7707 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7708 DPFC_NUKE_ON_ANY_MODIFICATION);
7711 * WaFbcHighMemBwCorruptionAvoidance:skl
7712 * Display WA #0883: skl
7714 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7715 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7716 DPFC_DISABLE_DUMMY0);
7719 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7723 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7724 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7725 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7728 /* WaSwitchSolVfFArbitrationPriority:bdw */
7729 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7731 /* WaPsrDPAMaskVBlankInSRD:bdw */
7732 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7733 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7735 for_each_pipe(dev_priv, pipe) {
7736 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7737 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7738 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7739 BDW_DPRS_MASK_VBLANK_SRD);
7742 /* WaVSRefCountFullforceMissDisable:bdw */
7743 /* WaDSRefCountFullforceMissDisable:bdw */
7744 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7745 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7746 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7748 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7749 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7751 /* WaDisableSDEUnitClockGating:bdw */
7752 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7753 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7755 /* WaProgramL3SqcReg1Default:bdw */
7756 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7758 /* WaKVMNotificationOnConfigChange:bdw */
7759 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7760 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7762 lpt_init_clock_gating(dev_priv);
7764 /* WaDisableDopClockGating:bdw
7766 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7769 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7770 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7773 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7775 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7776 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7777 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7780 /* This is required by WaCatErrorRejectionIssue:hsw */
7781 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7782 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7783 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7785 /* WaSwitchSolVfFArbitrationPriority:hsw */
7786 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7788 lpt_init_clock_gating(dev_priv);
7791 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7795 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7797 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7798 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7799 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7802 /* WaDisableBackToBackFlipFix:ivb */
7803 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7804 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7805 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7807 if (IS_IVB_GT1(dev_priv))
7808 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7809 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7811 /* must write both registers */
7812 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7813 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7814 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7815 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7819 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7820 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7822 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7823 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7825 /* This is required by WaCatErrorRejectionIssue:ivb */
7826 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7827 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7828 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7830 g4x_disable_trickle_feed(dev_priv);
7832 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7833 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7834 snpcr |= GEN6_MBC_SNPCR_MED;
7835 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7837 if (!HAS_PCH_NOP(dev_priv))
7838 cpt_init_clock_gating(dev_priv);
7840 gen6_check_mch_setup(dev_priv);
7843 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7845 /* WaDisableBackToBackFlipFix:vlv */
7846 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7847 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7848 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7850 /* WaDisableDopClockGating:vlv */
7851 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7852 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7854 /* This is required by WaCatErrorRejectionIssue:vlv */
7855 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7856 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7857 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7860 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7861 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7863 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7864 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7866 /* WaDisableL3Bank2xClockGate:vlv
7867 * Disabling L3 clock gating- MMIO 940c[25] = 1
7868 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7869 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7870 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7873 * WaDisableVLVClockGating_VBIIssue:vlv
7874 * Disable clock gating on th GCFG unit to prevent a delay
7875 * in the reporting of vblank events.
7877 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7880 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7882 /* WaVSRefCountFullforceMissDisable:chv */
7883 /* WaDSRefCountFullforceMissDisable:chv */
7884 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7885 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7886 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7888 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7889 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7890 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7892 /* WaDisableCSUnitClockGating:chv */
7893 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7894 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7896 /* WaDisableSDEUnitClockGating:chv */
7897 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7898 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7901 * WaProgramL3SqcReg1Default:chv
7902 * See gfxspecs/Related Documents/Performance Guide/
7903 * LSQC Setting Recommendations.
7905 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7908 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7912 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7913 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7914 GS_UNIT_CLOCK_GATE_DISABLE |
7915 CL_UNIT_CLOCK_GATE_DISABLE);
7916 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7917 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7918 OVRUNIT_CLOCK_GATE_DISABLE |
7919 OVCUNIT_CLOCK_GATE_DISABLE;
7920 if (IS_GM45(dev_priv))
7921 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7922 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7924 g4x_disable_trickle_feed(dev_priv);
7927 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7929 struct intel_uncore *uncore = &dev_priv->uncore;
7931 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7932 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7933 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7934 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7935 intel_uncore_write16(uncore, DEUC, 0);
7936 intel_uncore_write(uncore,
7938 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7941 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7943 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7944 I965_RCC_CLOCK_GATE_DISABLE |
7945 I965_RCPB_CLOCK_GATE_DISABLE |
7946 I965_ISC_CLOCK_GATE_DISABLE |
7947 I965_FBC_CLOCK_GATE_DISABLE);
7948 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7949 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7950 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7953 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7955 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7957 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7958 DSTATE_DOT_CLOCK_GATING;
7959 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7961 if (IS_PINEVIEW(dev_priv))
7962 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
7963 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7965 /* IIR "flip pending" means done if this bit is set */
7966 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
7967 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7969 /* interrupts should cause a wake up from C3 */
7970 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7972 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7973 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7975 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7976 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7979 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7981 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7983 /* interrupts should cause a wake up from C3 */
7984 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7985 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7987 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7988 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7991 * Have FBC ignore 3D activity since we use software
7992 * render tracking, and otherwise a pure 3D workload
7993 * (even if it just renders a single frame and then does
7994 * abosultely nothing) would not allow FBC to recompress
7995 * until a 2D blit occurs.
7997 intel_uncore_write(&dev_priv->uncore, SCPD0,
7998 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
8001 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
8003 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
8004 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
8005 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
8008 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
8010 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
8013 void intel_suspend_hw(struct drm_i915_private *dev_priv)
8015 if (HAS_PCH_LPT(dev_priv))
8016 lpt_suspend_hw(dev_priv);
8019 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
8021 drm_dbg_kms(&dev_priv->drm,
8022 "No clock gating settings or workarounds applied.\n");
8025 #define CG_FUNCS(platform) \
8026 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
8027 .init_clock_gating = platform##_init_clock_gating, \
8058 * intel_init_clock_gating_hooks - setup the clock gating hooks
8059 * @dev_priv: device private
8061 * Setup the hooks that configure which clocks of a given platform can be
8062 * gated and also apply various GT and display specific workarounds for these
8063 * platforms. Note that some GT specific workarounds are applied separately
8064 * when GPU contexts or batchbuffers start their execution.
8066 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
8068 if (IS_DG2(dev_priv))
8069 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
8070 else if (IS_XEHPSDV(dev_priv))
8071 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
8072 else if (IS_ALDERLAKE_P(dev_priv))
8073 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
8074 else if (IS_DG1(dev_priv))
8075 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
8076 else if (GRAPHICS_VER(dev_priv) == 12)
8077 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
8078 else if (GRAPHICS_VER(dev_priv) == 11)
8079 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
8080 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
8081 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
8082 else if (IS_SKYLAKE(dev_priv))
8083 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
8084 else if (IS_KABYLAKE(dev_priv))
8085 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
8086 else if (IS_BROXTON(dev_priv))
8087 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
8088 else if (IS_GEMINILAKE(dev_priv))
8089 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
8090 else if (IS_BROADWELL(dev_priv))
8091 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
8092 else if (IS_CHERRYVIEW(dev_priv))
8093 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
8094 else if (IS_HASWELL(dev_priv))
8095 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
8096 else if (IS_IVYBRIDGE(dev_priv))
8097 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
8098 else if (IS_VALLEYVIEW(dev_priv))
8099 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
8100 else if (GRAPHICS_VER(dev_priv) == 6)
8101 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
8102 else if (GRAPHICS_VER(dev_priv) == 5)
8103 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
8104 else if (IS_G4X(dev_priv))
8105 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
8106 else if (IS_I965GM(dev_priv))
8107 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
8108 else if (IS_I965G(dev_priv))
8109 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
8110 else if (GRAPHICS_VER(dev_priv) == 3)
8111 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
8112 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
8113 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
8114 else if (GRAPHICS_VER(dev_priv) == 2)
8115 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
8117 MISSING_CASE(INTEL_DEVID(dev_priv));
8118 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
8122 static const struct drm_i915_wm_disp_funcs skl_wm_funcs = {
8123 .compute_global_watermarks = skl_compute_wm,
8126 static const struct drm_i915_wm_disp_funcs ilk_wm_funcs = {
8127 .compute_pipe_wm = ilk_compute_pipe_wm,
8128 .compute_intermediate_wm = ilk_compute_intermediate_wm,
8129 .initial_watermarks = ilk_initial_watermarks,
8130 .optimize_watermarks = ilk_optimize_watermarks,
8133 static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = {
8134 .compute_pipe_wm = vlv_compute_pipe_wm,
8135 .compute_intermediate_wm = vlv_compute_intermediate_wm,
8136 .initial_watermarks = vlv_initial_watermarks,
8137 .optimize_watermarks = vlv_optimize_watermarks,
8138 .atomic_update_watermarks = vlv_atomic_update_fifo,
8141 static const struct drm_i915_wm_disp_funcs g4x_wm_funcs = {
8142 .compute_pipe_wm = g4x_compute_pipe_wm,
8143 .compute_intermediate_wm = g4x_compute_intermediate_wm,
8144 .initial_watermarks = g4x_initial_watermarks,
8145 .optimize_watermarks = g4x_optimize_watermarks,
8148 static const struct drm_i915_wm_disp_funcs pnv_wm_funcs = {
8149 .update_wm = pnv_update_wm,
8152 static const struct drm_i915_wm_disp_funcs i965_wm_funcs = {
8153 .update_wm = i965_update_wm,
8156 static const struct drm_i915_wm_disp_funcs i9xx_wm_funcs = {
8157 .update_wm = i9xx_update_wm,
8160 static const struct drm_i915_wm_disp_funcs i845_wm_funcs = {
8161 .update_wm = i845_update_wm,
8164 static const struct drm_i915_wm_disp_funcs nop_funcs = {
8167 /* Set up chip specific power management-related functions */
8168 void intel_init_pm(struct drm_i915_private *dev_priv)
8171 if (IS_PINEVIEW(dev_priv))
8172 pnv_get_mem_freq(dev_priv);
8173 else if (GRAPHICS_VER(dev_priv) == 5)
8174 ilk_get_mem_freq(dev_priv);
8176 if (intel_has_sagv(dev_priv))
8177 skl_setup_sagv_block_time(dev_priv);
8179 /* For FIFO watermark updates */
8180 if (DISPLAY_VER(dev_priv) >= 9) {
8181 skl_setup_wm_latency(dev_priv);
8182 dev_priv->wm_disp = &skl_wm_funcs;
8183 } else if (HAS_PCH_SPLIT(dev_priv)) {
8184 ilk_setup_wm_latency(dev_priv);
8186 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
8187 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
8188 (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
8189 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
8190 dev_priv->wm_disp = &ilk_wm_funcs;
8192 drm_dbg_kms(&dev_priv->drm,
8193 "Failed to read display plane latency. "
8195 dev_priv->wm_disp = &nop_funcs;
8197 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8198 vlv_setup_wm_latency(dev_priv);
8199 dev_priv->wm_disp = &vlv_wm_funcs;
8200 } else if (IS_G4X(dev_priv)) {
8201 g4x_setup_wm_latency(dev_priv);
8202 dev_priv->wm_disp = &g4x_wm_funcs;
8203 } else if (IS_PINEVIEW(dev_priv)) {
8204 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
8207 dev_priv->mem_freq)) {
8208 drm_info(&dev_priv->drm,
8209 "failed to find known CxSR latency "
8210 "(found ddr%s fsb freq %d, mem freq %d), "
8212 (dev_priv->is_ddr3 == 1) ? "3" : "2",
8213 dev_priv->fsb_freq, dev_priv->mem_freq);
8214 /* Disable CxSR and never update its watermark again */
8215 intel_set_memory_cxsr(dev_priv, false);
8216 dev_priv->wm_disp = &nop_funcs;
8218 dev_priv->wm_disp = &pnv_wm_funcs;
8219 } else if (DISPLAY_VER(dev_priv) == 4) {
8220 dev_priv->wm_disp = &i965_wm_funcs;
8221 } else if (DISPLAY_VER(dev_priv) == 3) {
8222 dev_priv->wm_disp = &i9xx_wm_funcs;
8223 } else if (DISPLAY_VER(dev_priv) == 2) {
8224 if (INTEL_NUM_PIPES(dev_priv) == 1)
8225 dev_priv->wm_disp = &i845_wm_funcs;
8227 dev_priv->wm_disp = &i9xx_wm_funcs;
8229 drm_err(&dev_priv->drm,
8230 "unexpected fall-through in %s\n", __func__);
8231 dev_priv->wm_disp = &nop_funcs;
8235 void intel_pm_setup(struct drm_i915_private *dev_priv)
8237 dev_priv->runtime_pm.suspended = false;
8238 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8241 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8243 struct intel_dbuf_state *dbuf_state;
8245 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8249 return &dbuf_state->base;
8252 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8253 struct intel_global_state *state)
8258 static const struct intel_global_state_funcs intel_dbuf_funcs = {
8259 .atomic_duplicate_state = intel_dbuf_duplicate_state,
8260 .atomic_destroy_state = intel_dbuf_destroy_state,
8263 struct intel_dbuf_state *
8264 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8266 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8267 struct intel_global_state *dbuf_state;
8269 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8270 if (IS_ERR(dbuf_state))
8271 return ERR_CAST(dbuf_state);
8273 return to_intel_dbuf_state(dbuf_state);
8276 int intel_dbuf_init(struct drm_i915_private *dev_priv)
8278 struct intel_dbuf_state *dbuf_state;
8280 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8284 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8285 &dbuf_state->base, &intel_dbuf_funcs);
8291 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
8292 * update the request state of all DBUS slices.
8294 static void update_mbus_pre_enable(struct intel_atomic_state *state)
8296 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8297 u32 mbus_ctl, dbuf_min_tracker_val;
8298 enum dbuf_slice slice;
8299 const struct intel_dbuf_state *dbuf_state =
8300 intel_atomic_get_new_dbuf_state(state);
8302 if (!IS_ALDERLAKE_P(dev_priv))
8306 * TODO: Implement vblank synchronized MBUS joining changes.
8307 * Must be properly coordinated with dbuf reprogramming.
8309 if (dbuf_state->joined_mbus) {
8310 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8311 MBUS_JOIN_PIPE_SELECT_NONE;
8312 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8314 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8315 MBUS_JOIN_PIPE_SELECT_NONE;
8316 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8319 intel_de_rmw(dev_priv, MBUS_CTL,
8320 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8321 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8323 for_each_dbuf_slice(dev_priv, slice)
8324 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8325 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8326 dbuf_min_tracker_val);
8329 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8331 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8332 const struct intel_dbuf_state *new_dbuf_state =
8333 intel_atomic_get_new_dbuf_state(state);
8334 const struct intel_dbuf_state *old_dbuf_state =
8335 intel_atomic_get_old_dbuf_state(state);
8337 if (!new_dbuf_state ||
8338 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8339 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8342 WARN_ON(!new_dbuf_state->base.changed);
8344 update_mbus_pre_enable(state);
8345 gen9_dbuf_slices_update(dev_priv,
8346 old_dbuf_state->enabled_slices |
8347 new_dbuf_state->enabled_slices);
8350 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8352 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8353 const struct intel_dbuf_state *new_dbuf_state =
8354 intel_atomic_get_new_dbuf_state(state);
8355 const struct intel_dbuf_state *old_dbuf_state =
8356 intel_atomic_get_old_dbuf_state(state);
8358 if (!new_dbuf_state ||
8359 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8360 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8363 WARN_ON(!new_dbuf_state->base.changed);
8365 gen9_dbuf_slices_update(dev_priv,
8366 new_dbuf_state->enabled_slices);